diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index f35b7212..9df8fd76 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -480,6 +480,25 @@ func (r *taskPendingOpsRepository) PendingCount( return n, nil } +// ClaimableCount uses ClaimBatch's eligibility and grouping rules so running +// documents and multiple queued operations for one document do not inflate +// the number of batches a scheduler needs to dispatch. +func (r *taskPendingOpsRepository) ClaimableCount( + ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time, +) (int64, error) { + freshKeys := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}). + Select("dedup_key"). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore) + var n int64 + err := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + Where("claimed_at IS NULL OR claimed_at < ?", staleBefore). + Where("dedup_key NOT IN (?)", freshKeys). + Distinct("dedup_key").Count(&n).Error + return n, err +} + // DeleteByDedupKey drops rows in the tuple whose dedup_key matches. // If `op` is non-empty, only rows with the matching op are dropped; // otherwise every matching row is removed. Empty dedup_key is rejected diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index dc8fe547..08aaafd9 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -1037,3 +1037,44 @@ func TestTaskPendingOps_DrainUnclaimedAndReleaseRollsBackOnReleaseFailure(t *tes require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error) assert.Equal(t, int64(1), count) } + +func TestTaskPendingOps_ClaimableCountMatchesClaimBatch(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + counter := repo.(interfaces.TaskPendingOpsClaimableCounter) + ctx := context.Background() + staleBefore := time.Now().Add(-time.Hour) + for _, key := range []string{"ready", "ready", "stale", "stale", "busy", "busy"} { + require.NoError(t, repo.Enqueue(ctx, + makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", key, nil))) + } + require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "stale"). + Update("claimed_at", staleBefore.Add(-time.Second)).Error) + require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "busy"). + Update("claimed_at", time.Now()).Error) + // An unclaimed late sibling of a busy key is not eligible on its own. + require.NoError(t, repo.Enqueue(ctx, + makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "retract", "busy", nil))) + // Identical keys in another tuple must not exclude the ready document. + other := makePendingOp("wiki:ingest", "knowledge_base", "other-kb", "ingest", "ready", nil) + require.NoError(t, repo.Enqueue(ctx, other)) + require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("scope_id = ?", "other-kb"). + Update("claimed_at", time.Now()).Error) + n, err := counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore) + require.NoError(t, err) + require.Equal(t, int64(2), n) + rows, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb-1", 2, staleBefore) + require.NoError(t, err) + require.Len(t, rows, 4) + keys := map[string]bool{} + for _, row := range rows { + keys[row.DedupKey] = true + } + require.Len(t, keys, int(n)) + n, err = counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore) + require.NoError(t, err) + require.Zero(t, n) + pending, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-1") + require.NoError(t, err) + require.Equal(t, int64(7), pending, "crash recovery must still see claimed rows") +} diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 796f7c0a..e9c74397 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -18,26 +18,16 @@ import ( "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" "github.com/hibiken/asynq" "golang.org/x/sync/errgroup" ) -// followUpTriggerCount decides how many follow-up triggers a completed batch -// should re-arm for its KB (#3461). Since Phase 3 the consumption side -// (disjoint claiming, per-slug locks, the inflight cap) fully supports -// concurrent batches, but nothing created more than one trigger for a KB with -// a standing backlog — so once uploads stopped, the backlog drained at -// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving -// the wiki pool and the model server idle. -// -// The count is the smaller of (a) how many batches the remaining rows still -// need and (b) how many slots could be free by the time the triggers fire: -// the caller's own slot releases right after it returns, hence the +1. -// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight -// <= 1 cannot host more than one batch anyway; both collapse to 1, matching -// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that -// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID. +// followUpTriggerCount estimates useful new work from claimable documents and +// running slots. reserveFollowUpTasks additionally bounds all outstanding +// follow-ups (scheduled, queued, running and retrying) across batch completions. +// The +1 allows the completing batch to replace itself before releasing its slot. func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int { if pending <= 0 || batchSize <= 0 { return 0 @@ -53,31 +43,9 @@ func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int return min(batchesNeeded, freeSlots) } -// scheduleFollowUp enqueues asynq trigger tasks if there are still pending -// ops in task_pending_ops for this KB. Returns true when at least one -// follow-up was scheduled. -// -// Post-Phase-3 this backstops the case where a batch drained its claimed -// window but more rows remain and no other trigger is pending (e.g. a steady -// trickle of uploads). Standard mode already fans a KB's backlog across -// concurrent claiming batches, so the short delay is normally just a light -// debounce rather than a lock-release wait. -// -// How many triggers are re-armed follows followUpTriggerCount above: enough -// to drain the backlog, capped by the free inflight slots, so the concurrent -// batch count converges on ingest_max_inflight instead of idling at 1. -// -// Fan-out is deliberately skipped when rateLimited: widening concurrency -// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also -// never fans out — liteLocks serializes batches per KB, so extra triggers -// would only bounce off ErrWikiIngestConcurrent. -// -// `delay` is the ProcessIn before each follow-up fires. Callers pass -// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the -// batch tripped an upstream rate limit — the released failed rows are -// eligible immediately, but nothing claims them until a trigger fires, so -// stretching the follow-up interval is what actually paces retries down -// during a 429 storm. +// scheduleFollowUp replenishes the KB's bounded supply of follow-up tasks. +// Lite mode and rate-limited completions request only one successor. A 429 +// delays this batch's successor; it does not pause other batches in the KB. func (s *wikiIngestService) scheduleFollowUp( ctx context.Context, payload WikiIngestPayload, @@ -93,38 +61,74 @@ func (s *wikiIngestService) scheduleFollowUp( return false } + if s.redisClient != nil { + if counter, ok := s.pendingRepo.(interfaces.TaskPendingOpsClaimableCounter); ok { + count, err = counter.ClaimableCount(ctx, wikiTaskType, wikiTaskScope, + payload.KnowledgeBaseID, time.Now().Add(-wikiClaimStaleAfter)) + if err != nil { + logger.Warnf(ctx, "wiki ingest: claimable count failed: %v", err) + return s.scheduleFollowUpRecheck(ctx, payload) + } + if count == 0 { + return s.scheduleStaleClaimRecheck(ctx, payload) + } + } + // ClaimBatch caps the number of distinct keys at 1000. + batchSize = min(batchSize, 1000) + } + // activeSlots stays -1 (unknown → single follow-up) in Lite mode and on - // a rate-limited exit; -1 also shows up in the log line as "unknown". + // a rate-limited exit. activeSlots := -1 if !rateLimited { activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID) } n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight) - activeStr := fmt.Sprintf("%d", activeSlots) - if activeSlots < 0 { - activeStr = "unknown" + var taskIDs []string + if s.redisClient != nil { + taskIDs, err = s.reserveFollowUpTasks(ctx, payload.KnowledgeBaseID, n, maxInflight) + if err != nil { + logger.Warnf(ctx, "wiki ingest: follow-up reservation failed: %v", err) + // Keep the pre-fan-out behavior while Redis is unavailable. + n = 1 + } else { + n = len(taskIDs) + if n == 0 { + return s.scheduleFollowUpRecheck(ctx, payload) + } + } } - logger.Infof(ctx, - "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)", - count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight) + logger.Infof(ctx, "wiki ingest: %d claimable documents for KB %s, scheduling %d follow-up(s) in %s", + count, payload.KnowledgeBaseID, n, delay) langfuse.InjectTracing(ctx, &payload) payloadBytes, _ := json.Marshal(payload) scheduled := false + needsRecheck := false for i := 0; i < n; i++ { - t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, + opts := []asynq.Option{ asynq.Queue(types.QueueWiki), asynq.MaxRetry(wikiIngestMaxRetry), - asynq.Timeout(60*time.Minute), - asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder - ) + asynq.Timeout(60 * time.Minute), + asynq.ProcessIn(delay), + } + if len(taskIDs) > 0 { + opts = append(opts, asynq.TaskID(taskIDs[i])) + } + t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, opts...) if _, err := s.task.Enqueue(t); err != nil { logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err) + // A timeout may mean the enqueue succeeded. Keep its reservation + // until reconciliation can establish the actual queue state. + needsRecheck = true continue } scheduled = true } + if needsRecheck { + scheduled = s.scheduleFollowUpRecheck(ctx, payload) || scheduled + } return scheduled } @@ -253,7 +257,7 @@ func (s *wikiIngestService) newWikiBatchContext( } } -func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error { +func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) (retErr error) { taskStartedAt := time.Now() retryCount, _ := asynq.GetRetryCount(ctx) maxRetry, _ := asynq.GetMaxRetry(ctx) @@ -312,6 +316,16 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task return fmt.Errorf("wiki ingest: unmarshal payload: %w", err) } + defer func() { + if p := recover(); p != nil { + // Asynq retries panics too: retain the outstanding reservation. + panic(p) + } + if retErr == nil { + s.releaseFollowUpTask(ctx, payload.KnowledgeBaseID) + } + }() + // Inject context ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { diff --git a/internal/application/service/wiki_ingest_followup.go b/internal/application/service/wiki_ingest_followup.go new file mode 100644 index 00000000..3bcd9675 --- /dev/null +++ b/internal/application/service/wiki_ingest_followup.go @@ -0,0 +1,152 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/tracing/langfuse" + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" +) + +const ( + wikiFollowUpPrefix = "wiki:followup:" + // This is a reconciliation interval, not a queue lease. A task can wait + // arbitrarily long for a worker without losing its reserved capacity. + // Missing tasks are retained for this interval to allow an enqueue in + // progress to finish, including when its caller times out. + wikiFollowUpReconcileAfter = 2 * time.Minute +) + +// Reservations cover the entire asynq task lifetime, including delayed tasks +// and retries. Keep one spare beyond the running cap so the last completing +// task can enqueue its successor before its own reservation is released. +// Atomic ZCARD + ZADD makes concurrent completions share the same budget. +const wikiFollowUpReserveScript = ` +local available = math.max(0, tonumber(ARGV[2]) - redis.call('ZCARD', KEYS[1])) +local n = math.min(tonumber(ARGV[1]), available) +local ids = {} +for i = 1, n do + local id = ARGV[3] .. ':' .. i + redis.call('ZADD', KEYS[1], ARGV[4], id) + ids[#ids + 1] = id +end +return ids +` + +// Do not expire a reservation just because the task has waited a long time. +// Inspect asynq first, then remove/refresh only if another reconciler hasn't +// already updated it. ZADD XX also avoids resurrecting a completed task. +const wikiFollowUpReconcileScript = ` +local score = redis.call('ZSCORE', KEYS[1], ARGV[1]) +if score and tonumber(score) == tonumber(ARGV[2]) then + if ARGV[3] == 'remove' then + redis.call('ZREM', KEYS[1], ARGV[1]) + else + redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1]) + end +end +return 1 +` + +func (s *wikiIngestService) reserveFollowUpTasks( + ctx context.Context, kbID string, requested, maxInflight int, +) ([]string, error) { + if requested <= 0 { + return nil, nil + } + if err := s.reconcileFollowUpTasks(ctx, kbID); err != nil { + return nil, err + } + return s.redisClient.Eval(ctx, wikiFollowUpReserveScript, + []string{wikiFollowUpPrefix + kbID}, requested, max(1, maxInflight)+1, + "wiki-followup-"+uuid.NewString(), time.Now().Add(wikiFollowUpReconcileAfter).UnixMilli(), + ).StringSlice() +} + +func (s *wikiIngestService) reconcileFollowUpTasks(ctx context.Context, kbID string) error { + key := wikiFollowUpPrefix + kbID + now := time.Now() + entries, err := s.redisClient.ZRangeByScoreWithScores(ctx, key, &redis.ZRangeBy{ + Min: "-inf", Max: strconv.FormatInt(now.UnixMilli(), 10), + }).Result() + if err != nil || len(entries) == 0 { + return err + } + // Shares the service's Redis connection; Inspector.Close must not close it. + inspector := asynq.NewInspectorFromRedisClient(s.redisClient) + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return err + } + id, ok := entry.Member.(string) + if !ok { + continue + } + info, err := inspector.GetTaskInfo(types.QueueWiki, id) + missing := errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) + if err != nil && !missing { + return err + } + action := strconv.FormatInt(now.Add(wikiFollowUpReconcileAfter).UnixMilli(), 10) + if missing || info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted { + action = "remove" + } + if err := s.redisClient.Eval(ctx, wikiFollowUpReconcileScript, + []string{key}, id, entry.Score, action).Err(); err != nil { + return err + } + } + return nil +} + +func (s *wikiIngestService) releaseFollowUpTask(ctx context.Context, kbID string) { + if s.redisClient == nil { + return + } + id, ok := asynq.GetTaskID(ctx) + if !ok { + return + } + cleanupCtx, cancel := wikiIngestCleanupContext(ctx) + defer cancel() + if err := s.redisClient.ZRem(cleanupCtx, wikiFollowUpPrefix+kbID, id).Err(); err != nil { + logger.Warnf(ctx, "wiki ingest: follow-up reservation release failed: %v", err) + } +} + +// A coalesced probe keeps progress possible when all capacity is reserved but +// an enqueue failed or a worker died. It also revisits reservations whose +// cleanup failed. It is outside the reservation budget and contributes at +// most two probes per KB, independent of the backlog size. +func (s *wikiIngestService) scheduleFollowUpRecheck(ctx context.Context, payload WikiIngestPayload) bool { + langfuse.InjectTracing(ctx, &payload) + b, _ := json.Marshal(payload) + id := "wiki-followup-recheck-" + payload.KnowledgeBaseID + if current, _ := asynq.GetTaskID(ctx); current == id { + // The current task ID remains occupied until asynq acknowledges it. + // Alternate between two IDs so the probe can re-arm itself. + id += "-next" + } + t := asynq.NewTask(types.TypeWikiIngest, b, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(60*time.Minute), + asynq.ProcessIn(wikiFollowUpReconcileAfter+wikiFollowUpDelay), + asynq.TaskID(id), + ) + if _, err := s.task.Enqueue(t); err != nil { + if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) { + return true + } + logger.Warnf(ctx, "wiki ingest: follow-up recheck enqueue failed: %v", err) + return false + } + return true +} diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.go index 2536661e..6657edab 100644 --- a/internal/application/service/wiki_ingest_followup_test.go +++ b/internal/application/service/wiki_ingest_followup_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "sync" "testing" "time" @@ -17,10 +18,12 @@ import ( // --- test doubles ----------------------------------------------------------- // wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes -// PendingCount configurable, which is the only input scheduleFollowUp reads. +// PendingCount and the eligible-document count configurable. type wikiPendingRepoForFollowUpTest struct { wikiPendingRepoForCleanupTest - pending int64 + pending int64 + claimable *int64 + claimableErr error } func (r *wikiPendingRepoForFollowUpTest) PendingCount( @@ -29,6 +32,15 @@ func (r *wikiPendingRepoForFollowUpTest) PendingCount( return r.pending, nil } +func (r *wikiPendingRepoForFollowUpTest) ClaimableCount( + context.Context, string, string, string, time.Time, +) (int64, error) { + if r.claimable != nil { + return *r.claimable, r.claimableErr + } + return r.pending, r.claimableErr +} + // followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues. // failFrom >= 0 makes enqueues from that index onward return failErr so the // partial-failure path can be exercised. @@ -248,47 +260,274 @@ func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T got := svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false) require.True(t, got) - require.Equal(t, 4, rec.count()) + require.Equal(t, 5, rec.count()) // four follow-ups and one recovery probe } -// --- convergence ------------------------------------------------------------ +// Use the actual asynq client so these tests exercise atomic reservations and +// queue state, rather than merely checking the single-completer arithmetic. +func newQueuedFollowUpTestService(t *testing.T) (*wikiIngestService, *asynq.Inspector) { + t.Helper() + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + client := asynq.NewClient(asynq.RedisClientOpt{Addr: mr.Addr()}) + t.Cleanup(func() { _ = client.Close() }) + return &wikiIngestService{ + redisClient: rdb, + task: client, + pendingRepo: &wikiPendingRepoForFollowUpTest{pending: 24624}, + }, asynq.NewInspectorFromRedisClient(rdb) +} -// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions -// fanning out on a board whose slots never free (worst case for over-issue) -// and asserts the per-completion trigger count never exceeds the free slots, -// i.e. the supply side alone cannot push concurrency past ingest_max_inflight. -func TestFollowUpFanOutConvergesUnderCap(t *testing.T) { - const ( - maxInflight = 8 - batchSize = 10 - ) - var ( - pending = 1000 - activeSlots = 1 // the completing batch itself; releases on return - ) - seen := map[int]bool{} - for round := 0; round < 6; round++ { - n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight) - require.Greater(t, n, 0) - require.LessOrEqual(t, n, maxInflight-activeSlots+1, - "round %d: supply must not exceed free slots", round) - if activeSlots == maxInflight { - require.Equal(t, 1, n, - "round %d: a full board must clamp supply to the follow-up chain", round) - } - seen[n] = true - - // The n triggers fire: n-1 of them claim slots (one bounces off the - // cap into scheduleCappedRetry when the board is full), each claims a - // batch of batchSize rows. - newBatches := min(n, maxInflight-activeSlots) - pending -= newBatches * batchSize - activeSlots = maxInflight - if pending <= 0 { - pending = 1 // keep the loop meaningful; count stays capped below - } +func expireFollowUpChecks(t *testing.T, svc *wikiIngestService) { + t.Helper() + ctx := context.Background() + ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result() + require.NoError(t, err) + for _, id := range ids { + require.NoError(t, svc.redisClient.ZAdd(ctx, wikiFollowUpPrefix+"kb-1", redis.Z{ + Score: float64(time.Now().Add(-time.Minute).UnixMilli()), Member: id, + }).Err()) } - // The first round must have fanned out beyond the follow-up chain — - // otherwise this would not be testing convergence at all. - require.Greater(t, len(seen), 1) +} + +func TestScheduleFollowUpWorkerPoolBelowCap(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + ctx := context.Background() + payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"} + // Startup recovery has one running batch, whose completion fans out. + release, ok := svc.reserveInflightSlot(ctx, "kb-1", 32) + require.True(t, ok) + require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false)) + release() + queued, err := inspector.ListScheduledTasks("wiki", asynq.PageSize(100)) + require.NoError(t, err) + require.Len(t, queued, 32) + + // Model eight workers dequeuing tasks. Keep their follow-up reservations + // until completion, just as the handler does, and consume one queued task + // for each finished batch. No model, database or external Redis is used. + runningIDs := make([]string, 8) + for i := range runningIDs { + runningIDs[i] = queued[i].ID + require.NoError(t, inspector.DeleteTask("wiki", queued[i].ID)) + } + reserveSlots(t, svc, "kb-1", 8, 32) + for completed := 0; completed < 100; completed++ { + svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending -= 10 + require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false)) + worker := completed % len(runningIDs) + require.NoError(t, svc.redisClient.ZRem(ctx, wikiFollowUpPrefix+"kb-1", runningIDs[worker]).Err()) + queued, err = inspector.ListScheduledTasks("wiki", asynq.PageSize(100)) + require.NoError(t, err) + require.NotEmpty(t, queued) + runningIDs[worker] = queued[0].ID + require.NoError(t, inspector.DeleteTask("wiki", queued[0].ID)) + info, err := inspector.GetQueueInfo("wiki") + require.NoError(t, err) + require.LessOrEqual(t, info.Scheduled, 25, "completion %d must not multiply the queue", completed) + } +} + +func TestScheduleFollowUpConcurrentCompletionsShareBudget(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + reserveSlots(t, svc, "kb-1", 16, 32) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"}, + wikiFollowUpDelay, 10, 32, false) + }() + } + close(start) + wg.Wait() + info, err := inspector.GetQueueInfo("wiki") + require.NoError(t, err) + // Cap + one successor spare + at most one coalesced recovery probe. + require.LessOrEqual(t, info.Scheduled, 34) + require.Equal(t, int64(33), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val()) +} + +func TestScheduleFollowUpClaimedRowsOnlyKeepsCrashRecovery(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + zero := int64(0) + svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).claimable = &zero + require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"}, + wikiFollowUpDelay, 10, 32, false)) + tasks, err := inspector.ListScheduledTasks("wiki") + require.NoError(t, err) + require.Len(t, tasks, 1) + require.Equal(t, "wiki-ingest-recheck-kb-1", tasks[0].ID) + require.Greater(t, time.Until(tasks[0].NextProcessAt), wikiClaimStaleAfter-time.Second) +} + +func TestFollowUpReservationsKeepLongQueuedTasks(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + ctx := context.Background() + payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"} + require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false)) + for i := 0; i < 5; i++ { + expireFollowUpChecks(t, svc) + require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false)) + } + info, err := inspector.GetQueueInfo("wiki") + require.NoError(t, err) + require.LessOrEqual(t, info.Scheduled, 6) // five reserved tasks plus one probe + require.Equal(t, int64(5), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val()) +} + +func TestFollowUpReservationsRecoverMissingAndArchivedTasks(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + ctx := context.Background() + require.True(t, svc.scheduleFollowUp(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}, + wikiFollowUpDelay, 10, 4, false)) + tasks, err := inspector.ListScheduledTasks("wiki") + require.NoError(t, err) + require.NoError(t, inspector.DeleteTask("wiki", tasks[0].ID)) + require.NoError(t, inspector.ArchiveTask("wiki", tasks[1].ID)) + expireFollowUpChecks(t, svc) + require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1")) + ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result() + require.NoError(t, err) + require.NotContains(t, ids, tasks[0].ID) + require.NotContains(t, ids, tasks[1].ID) + require.Len(t, ids, len(tasks)-2) +} + +func TestFollowUpReservationsProtectEnqueueInProgress(t *testing.T) { + svc, _ := newQueuedFollowUpTestService(t) + ctx := context.Background() + ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4) + require.NoError(t, err) + require.Len(t, ids, 5) + // The task does not exist yet. Another completer must not recycle its + // capacity while the first enqueue is still in flight. + more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4) + require.NoError(t, err) + require.Empty(t, more) + // A producer that died before enqueueing must not consume capacity forever. + expireFollowUpChecks(t, svc) + more, err = svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4) + require.NoError(t, err) + require.Len(t, more, 5) +} + +// An enqueue can be committed by Redis even when its response is lost. +type lostFollowUpEnqueueResponse struct{ client *asynq.Client } + +func (q lostFollowUpEnqueueResponse) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) { + if _, err := q.client.Enqueue(task, opts...); err != nil { + return nil, err + } + return nil, errors.New("enqueue response lost") +} + +func TestFollowUpReservationsSurviveAmbiguousEnqueueFailure(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + client := svc.task.(*asynq.Client) + svc.task = lostFollowUpEnqueueResponse{client: client} + svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"}, + wikiFollowUpDelay, 10, 4, false) + svc.task = client + expireFollowUpChecks(t, svc) + require.NoError(t, svc.reconcileFollowUpTasks(context.Background(), "kb-1")) + require.Equal(t, int64(5), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val()) + info, err := inspector.GetQueueInfo("wiki") + require.NoError(t, err) + require.Equal(t, 6, info.Scheduled) // five real tasks and the recovery probe +} + +func TestFollowUpReservationHandlerLifecycle(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("retry=%t", fail), func(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + kbService := &wikiGuardKBService{} + if fail { + kbService.err = errors.New("temporary KB lookup failure") + } + svc.kbService = kbService + ctx := context.Background() + ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 1, 1) + require.NoError(t, err) + require.Len(t, ids, 1) + server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{ + Concurrency: 1, Queues: map[string]int{"wiki": 1}, + TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second, + RetryDelayFunc: func(int, error, *asynq.Task) time.Duration { return time.Hour }, + LogLevel: asynq.ErrorLevel, + }) + require.NoError(t, server.Start(asynq.HandlerFunc(svc.ProcessWikiIngest))) + t.Cleanup(server.Shutdown) + payload, err := json.Marshal(WikiIngestPayload{KnowledgeBaseID: "kb-1"}) + require.NoError(t, err) + _, err = svc.task.Enqueue(asynq.NewTask("wiki:ingest", payload, + asynq.Queue("wiki"), asynq.TaskID(ids[0]))) + require.NoError(t, err) + if fail { + require.Eventually(t, func() bool { + info, err := inspector.GetTaskInfo("wiki", ids[0]) + return err == nil && info.State == asynq.TaskStateRetry + }, 5*time.Second, 10*time.Millisecond) + expireFollowUpChecks(t, svc) + require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1")) + require.Equal(t, int64(1), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val()) + } else { + require.Eventually(t, func() bool { + return svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val() == 0 + }, 5*time.Second, 10*time.Millisecond, "successful handler must release its reservation") + } + }) + } +} + +func TestFollowUpRecheckCanRearmItself(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{ + Concurrency: 1, Queues: map[string]int{"wiki": 1}, + TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second, + LogLevel: asynq.ErrorLevel, + }) + require.NoError(t, server.Start(asynq.HandlerFunc(func(ctx context.Context, _ *asynq.Task) error { + if !svc.scheduleFollowUpRecheck(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}) { + return errors.New("could not re-arm probe") + } + return nil + }))) + t.Cleanup(server.Shutdown) + _, err := svc.task.Enqueue(asynq.NewTask("wiki:ingest", nil, + asynq.Queue("wiki"), asynq.TaskID("wiki-followup-recheck-kb-1"))) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, err := inspector.GetTaskInfo("wiki", "wiki-followup-recheck-kb-1-next") + return err == nil && info.State == asynq.TaskStateScheduled + }, 5*time.Second, 10*time.Millisecond) +} + +func TestFollowUpReservationsKeepCapacitySeparateForEachKB(t *testing.T) { + svc, _ := newQueuedFollowUpTestService(t) + ctx := context.Background() + first, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4) + require.NoError(t, err) + require.Len(t, first, 5) + second, err := svc.reserveFollowUpTasks(ctx, "kb-2", 5, 4) + require.NoError(t, err) + require.Len(t, second, 5) + more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4) + require.NoError(t, err) + require.Empty(t, more) +} + +func TestScheduleFollowUpUsesRepositoryBatchLimit(t *testing.T) { + svc, inspector := newQueuedFollowUpTestService(t) + svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending = 1500 + reserveSlots(t, svc, "kb-1", 1, 4) + require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"}, + wikiFollowUpDelay, 2000, 4, false)) + info, err := inspector.GetQueueInfo("wiki") + require.NoError(t, err) + require.Equal(t, 2, info.Scheduled, "ClaimBatch can claim at most 1000 documents per batch") } diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index e6d8dfd7..c50eb324 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -86,6 +86,13 @@ type TaskPendingOpsRepository interface { DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error } +// TaskPendingOpsClaimableCounter counts the distinct keys ClaimBatch can +// claim, excluding every key with a fresh claim (including late siblings). +// PendingCount remains the count of all rows for crash-recovery consumers. +type TaskPendingOpsClaimableCounter interface { + ClaimableCount(ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time) (int64, error) +} + // TaskPendingOpsScopeCleaner is an optional extension for callers that need // to discard every durable pending operation owned by a deleted scope. It is // intentionally separate from TaskPendingOpsRepository so alternate queue