fix(wiki): bound outstanding follow-up tasks

main
wizardchen 2026-09-28 21:17:40 +08:00 committed by lyingbug
parent 4ba85e2edf
commit c819abd958
6 changed files with 565 additions and 93 deletions

View File

@ -480,6 +480,25 @@ func (r *taskPendingOpsRepository) PendingCount(
return n, nil
}
// ClaimableCount uses ClaimBatch's eligibility and grouping rules so running
// documents and multiple queued operations for one document do not inflate
// the number of batches a scheduler needs to dispatch.
func (r *taskPendingOpsRepository) ClaimableCount(
ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time,
) (int64, error) {
freshKeys := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
Select("dedup_key").
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
var n int64
err := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
Where("claimed_at IS NULL OR claimed_at < ?", staleBefore).
Where("dedup_key NOT IN (?)", freshKeys).
Distinct("dedup_key").Count(&n).Error
return n, err
}
// DeleteByDedupKey drops rows in the tuple whose dedup_key matches.
// If `op` is non-empty, only rows with the matching op are dropped;
// otherwise every matching row is removed. Empty dedup_key is rejected

View File

@ -1037,3 +1037,44 @@ func TestTaskPendingOps_DrainUnclaimedAndReleaseRollsBackOnReleaseFailure(t *tes
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
assert.Equal(t, int64(1), count)
}
func TestTaskPendingOps_ClaimableCountMatchesClaimBatch(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
counter := repo.(interfaces.TaskPendingOpsClaimableCounter)
ctx := context.Background()
staleBefore := time.Now().Add(-time.Hour)
for _, key := range []string{"ready", "ready", "stale", "stale", "busy", "busy"} {
require.NoError(t, repo.Enqueue(ctx,
makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", key, nil)))
}
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "stale").
Update("claimed_at", staleBefore.Add(-time.Second)).Error)
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "busy").
Update("claimed_at", time.Now()).Error)
// An unclaimed late sibling of a busy key is not eligible on its own.
require.NoError(t, repo.Enqueue(ctx,
makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "retract", "busy", nil)))
// Identical keys in another tuple must not exclude the ready document.
other := makePendingOp("wiki:ingest", "knowledge_base", "other-kb", "ingest", "ready", nil)
require.NoError(t, repo.Enqueue(ctx, other))
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("scope_id = ?", "other-kb").
Update("claimed_at", time.Now()).Error)
n, err := counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
require.NoError(t, err)
require.Equal(t, int64(2), n)
rows, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb-1", 2, staleBefore)
require.NoError(t, err)
require.Len(t, rows, 4)
keys := map[string]bool{}
for _, row := range rows {
keys[row.DedupKey] = true
}
require.Len(t, keys, int(n))
n, err = counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
require.NoError(t, err)
require.Zero(t, n)
pending, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-1")
require.NoError(t, err)
require.Equal(t, int64(7), pending, "crash recovery must still see claimed rows")
}

View File

@ -18,26 +18,16 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"golang.org/x/sync/errgroup"
)
// followUpTriggerCount decides how many follow-up triggers a completed batch
// should re-arm for its KB (#3461). Since Phase 3 the consumption side
// (disjoint claiming, per-slug locks, the inflight cap) fully supports
// concurrent batches, but nothing created more than one trigger for a KB with
// a standing backlog — so once uploads stopped, the backlog drained at
// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
// the wiki pool and the model server idle.
//
// The count is the smaller of (a) how many batches the remaining rows still
// need and (b) how many slots could be free by the time the triggers fire:
// the caller's own slot releases right after it returns, hence the +1.
// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
// followUpTriggerCount estimates useful new work from claimable documents and
// running slots. reserveFollowUpTasks additionally bounds all outstanding
// follow-ups (scheduled, queued, running and retrying) across batch completions.
// The +1 allows the completing batch to replace itself before releasing its slot.
func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
if pending <= 0 || batchSize <= 0 {
return 0
@ -53,31 +43,9 @@ func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int
return min(batchesNeeded, freeSlots)
}
// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
// ops in task_pending_ops for this KB. Returns true when at least one
// follow-up was scheduled.
//
// Post-Phase-3 this backstops the case where a batch drained its claimed
// window but more rows remain and no other trigger is pending (e.g. a steady
// trickle of uploads). Standard mode already fans a KB's backlog across
// concurrent claiming batches, so the short delay is normally just a light
// debounce rather than a lock-release wait.
//
// How many triggers are re-armed follows followUpTriggerCount above: enough
// to drain the backlog, capped by the free inflight slots, so the concurrent
// batch count converges on ingest_max_inflight instead of idling at 1.
//
// Fan-out is deliberately skipped when rateLimited: widening concurrency
// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
// never fans out — liteLocks serializes batches per KB, so extra triggers
// would only bounce off ErrWikiIngestConcurrent.
//
// `delay` is the ProcessIn before each follow-up fires. Callers pass
// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
// batch tripped an upstream rate limit — the released failed rows are
// eligible immediately, but nothing claims them until a trigger fires, so
// stretching the follow-up interval is what actually paces retries down
// during a 429 storm.
// scheduleFollowUp replenishes the KB's bounded supply of follow-up tasks.
// Lite mode and rate-limited completions request only one successor. A 429
// delays this batch's successor; it does not pause other batches in the KB.
func (s *wikiIngestService) scheduleFollowUp(
ctx context.Context,
payload WikiIngestPayload,
@ -93,38 +61,74 @@ func (s *wikiIngestService) scheduleFollowUp(
return false
}
if s.redisClient != nil {
if counter, ok := s.pendingRepo.(interfaces.TaskPendingOpsClaimableCounter); ok {
count, err = counter.ClaimableCount(ctx, wikiTaskType, wikiTaskScope,
payload.KnowledgeBaseID, time.Now().Add(-wikiClaimStaleAfter))
if err != nil {
logger.Warnf(ctx, "wiki ingest: claimable count failed: %v", err)
return s.scheduleFollowUpRecheck(ctx, payload)
}
if count == 0 {
return s.scheduleStaleClaimRecheck(ctx, payload)
}
}
// ClaimBatch caps the number of distinct keys at 1000.
batchSize = min(batchSize, 1000)
}
// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
// a rate-limited exit; -1 also shows up in the log line as "unknown".
// a rate-limited exit.
activeSlots := -1
if !rateLimited {
activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
}
n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
activeStr := fmt.Sprintf("%d", activeSlots)
if activeSlots < 0 {
activeStr = "unknown"
var taskIDs []string
if s.redisClient != nil {
taskIDs, err = s.reserveFollowUpTasks(ctx, payload.KnowledgeBaseID, n, maxInflight)
if err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up reservation failed: %v", err)
// Keep the pre-fan-out behavior while Redis is unavailable.
n = 1
} else {
n = len(taskIDs)
if n == 0 {
return s.scheduleFollowUpRecheck(ctx, payload)
}
}
}
logger.Infof(ctx,
"wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
logger.Infof(ctx, "wiki ingest: %d claimable documents for KB %s, scheduling %d follow-up(s) in %s",
count, payload.KnowledgeBaseID, n, delay)
langfuse.InjectTracing(ctx, &payload)
payloadBytes, _ := json.Marshal(payload)
scheduled := false
needsRecheck := false
for i := 0; i < n; i++ {
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
opts := []asynq.Option{
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
)
asynq.Timeout(60 * time.Minute),
asynq.ProcessIn(delay),
}
if len(taskIDs) > 0 {
opts = append(opts, asynq.TaskID(taskIDs[i]))
}
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, opts...)
if _, err := s.task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
// A timeout may mean the enqueue succeeded. Keep its reservation
// until reconciliation can establish the actual queue state.
needsRecheck = true
continue
}
scheduled = true
}
if needsRecheck {
scheduled = s.scheduleFollowUpRecheck(ctx, payload) || scheduled
}
return scheduled
}
@ -253,7 +257,7 @@ func (s *wikiIngestService) newWikiBatchContext(
}
}
func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {
func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) (retErr error) {
taskStartedAt := time.Now()
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
@ -312,6 +316,16 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
return fmt.Errorf("wiki ingest: unmarshal payload: %w", err)
}
defer func() {
if p := recover(); p != nil {
// Asynq retries panics too: retain the outstanding reservation.
panic(p)
}
if retErr == nil {
s.releaseFollowUpTask(ctx, payload.KnowledgeBaseID)
}
}()
// Inject context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {

View File

@ -0,0 +1,152 @@
package service
import (
"context"
"encoding/json"
"errors"
"strconv"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
const (
wikiFollowUpPrefix = "wiki:followup:"
// This is a reconciliation interval, not a queue lease. A task can wait
// arbitrarily long for a worker without losing its reserved capacity.
// Missing tasks are retained for this interval to allow an enqueue in
// progress to finish, including when its caller times out.
wikiFollowUpReconcileAfter = 2 * time.Minute
)
// Reservations cover the entire asynq task lifetime, including delayed tasks
// and retries. Keep one spare beyond the running cap so the last completing
// task can enqueue its successor before its own reservation is released.
// Atomic ZCARD + ZADD makes concurrent completions share the same budget.
const wikiFollowUpReserveScript = `
local available = math.max(0, tonumber(ARGV[2]) - redis.call('ZCARD', KEYS[1]))
local n = math.min(tonumber(ARGV[1]), available)
local ids = {}
for i = 1, n do
local id = ARGV[3] .. ':' .. i
redis.call('ZADD', KEYS[1], ARGV[4], id)
ids[#ids + 1] = id
end
return ids
`
// Do not expire a reservation just because the task has waited a long time.
// Inspect asynq first, then remove/refresh only if another reconciler hasn't
// already updated it. ZADD XX also avoids resurrecting a completed task.
const wikiFollowUpReconcileScript = `
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
if score and tonumber(score) == tonumber(ARGV[2]) then
if ARGV[3] == 'remove' then
redis.call('ZREM', KEYS[1], ARGV[1])
else
redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1])
end
end
return 1
`
func (s *wikiIngestService) reserveFollowUpTasks(
ctx context.Context, kbID string, requested, maxInflight int,
) ([]string, error) {
if requested <= 0 {
return nil, nil
}
if err := s.reconcileFollowUpTasks(ctx, kbID); err != nil {
return nil, err
}
return s.redisClient.Eval(ctx, wikiFollowUpReserveScript,
[]string{wikiFollowUpPrefix + kbID}, requested, max(1, maxInflight)+1,
"wiki-followup-"+uuid.NewString(), time.Now().Add(wikiFollowUpReconcileAfter).UnixMilli(),
).StringSlice()
}
func (s *wikiIngestService) reconcileFollowUpTasks(ctx context.Context, kbID string) error {
key := wikiFollowUpPrefix + kbID
now := time.Now()
entries, err := s.redisClient.ZRangeByScoreWithScores(ctx, key, &redis.ZRangeBy{
Min: "-inf", Max: strconv.FormatInt(now.UnixMilli(), 10),
}).Result()
if err != nil || len(entries) == 0 {
return err
}
// Shares the service's Redis connection; Inspector.Close must not close it.
inspector := asynq.NewInspectorFromRedisClient(s.redisClient)
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return err
}
id, ok := entry.Member.(string)
if !ok {
continue
}
info, err := inspector.GetTaskInfo(types.QueueWiki, id)
missing := errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound)
if err != nil && !missing {
return err
}
action := strconv.FormatInt(now.Add(wikiFollowUpReconcileAfter).UnixMilli(), 10)
if missing || info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {
action = "remove"
}
if err := s.redisClient.Eval(ctx, wikiFollowUpReconcileScript,
[]string{key}, id, entry.Score, action).Err(); err != nil {
return err
}
}
return nil
}
func (s *wikiIngestService) releaseFollowUpTask(ctx context.Context, kbID string) {
if s.redisClient == nil {
return
}
id, ok := asynq.GetTaskID(ctx)
if !ok {
return
}
cleanupCtx, cancel := wikiIngestCleanupContext(ctx)
defer cancel()
if err := s.redisClient.ZRem(cleanupCtx, wikiFollowUpPrefix+kbID, id).Err(); err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up reservation release failed: %v", err)
}
}
// A coalesced probe keeps progress possible when all capacity is reserved but
// an enqueue failed or a worker died. It also revisits reservations whose
// cleanup failed. It is outside the reservation budget and contributes at
// most two probes per KB, independent of the backlog size.
func (s *wikiIngestService) scheduleFollowUpRecheck(ctx context.Context, payload WikiIngestPayload) bool {
langfuse.InjectTracing(ctx, &payload)
b, _ := json.Marshal(payload)
id := "wiki-followup-recheck-" + payload.KnowledgeBaseID
if current, _ := asynq.GetTaskID(ctx); current == id {
// The current task ID remains occupied until asynq acknowledges it.
// Alternate between two IDs so the probe can re-arm itself.
id += "-next"
}
t := asynq.NewTask(types.TypeWikiIngest, b,
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiFollowUpReconcileAfter+wikiFollowUpDelay),
asynq.TaskID(id),
)
if _, err := s.task.Enqueue(t); err != nil {
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
return true
}
logger.Warnf(ctx, "wiki ingest: follow-up recheck enqueue failed: %v", err)
return false
}
return true
}

View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"testing"
"time"
@ -17,10 +18,12 @@ import (
// --- test doubles -----------------------------------------------------------
// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
// PendingCount configurable, which is the only input scheduleFollowUp reads.
// PendingCount and the eligible-document count configurable.
type wikiPendingRepoForFollowUpTest struct {
wikiPendingRepoForCleanupTest
pending int64
pending int64
claimable *int64
claimableErr error
}
func (r *wikiPendingRepoForFollowUpTest) PendingCount(
@ -29,6 +32,15 @@ func (r *wikiPendingRepoForFollowUpTest) PendingCount(
return r.pending, nil
}
func (r *wikiPendingRepoForFollowUpTest) ClaimableCount(
context.Context, string, string, string, time.Time,
) (int64, error) {
if r.claimable != nil {
return *r.claimable, r.claimableErr
}
return r.pending, r.claimableErr
}
// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
// failFrom >= 0 makes enqueues from that index onward return failErr so the
// partial-failure path can be exercised.
@ -248,47 +260,274 @@ func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.True(t, got)
require.Equal(t, 4, rec.count())
require.Equal(t, 5, rec.count()) // four follow-ups and one recovery probe
}
// --- convergence ------------------------------------------------------------
// Use the actual asynq client so these tests exercise atomic reservations and
// queue state, rather than merely checking the single-completer arithmetic.
func newQueuedFollowUpTestService(t *testing.T) (*wikiIngestService, *asynq.Inspector) {
t.Helper()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
client := asynq.NewClient(asynq.RedisClientOpt{Addr: mr.Addr()})
t.Cleanup(func() { _ = client.Close() })
return &wikiIngestService{
redisClient: rdb,
task: client,
pendingRepo: &wikiPendingRepoForFollowUpTest{pending: 24624},
}, asynq.NewInspectorFromRedisClient(rdb)
}
// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
// fanning out on a board whose slots never free (worst case for over-issue)
// and asserts the per-completion trigger count never exceeds the free slots,
// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
const (
maxInflight = 8
batchSize = 10
)
var (
pending = 1000
activeSlots = 1 // the completing batch itself; releases on return
)
seen := map[int]bool{}
for round := 0; round < 6; round++ {
n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
require.Greater(t, n, 0)
require.LessOrEqual(t, n, maxInflight-activeSlots+1,
"round %d: supply must not exceed free slots", round)
if activeSlots == maxInflight {
require.Equal(t, 1, n,
"round %d: a full board must clamp supply to the follow-up chain", round)
}
seen[n] = true
// The n triggers fire: n-1 of them claim slots (one bounces off the
// cap into scheduleCappedRetry when the board is full), each claims a
// batch of batchSize rows.
newBatches := min(n, maxInflight-activeSlots)
pending -= newBatches * batchSize
activeSlots = maxInflight
if pending <= 0 {
pending = 1 // keep the loop meaningful; count stays capped below
}
func expireFollowUpChecks(t *testing.T, svc *wikiIngestService) {
t.Helper()
ctx := context.Background()
ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
require.NoError(t, err)
for _, id := range ids {
require.NoError(t, svc.redisClient.ZAdd(ctx, wikiFollowUpPrefix+"kb-1", redis.Z{
Score: float64(time.Now().Add(-time.Minute).UnixMilli()), Member: id,
}).Err())
}
// The first round must have fanned out beyond the follow-up chain —
// otherwise this would not be testing convergence at all.
require.Greater(t, len(seen), 1)
}
func TestScheduleFollowUpWorkerPoolBelowCap(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
ctx := context.Background()
payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
// Startup recovery has one running batch, whose completion fans out.
release, ok := svc.reserveInflightSlot(ctx, "kb-1", 32)
require.True(t, ok)
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
release()
queued, err := inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
require.NoError(t, err)
require.Len(t, queued, 32)
// Model eight workers dequeuing tasks. Keep their follow-up reservations
// until completion, just as the handler does, and consume one queued task
// for each finished batch. No model, database or external Redis is used.
runningIDs := make([]string, 8)
for i := range runningIDs {
runningIDs[i] = queued[i].ID
require.NoError(t, inspector.DeleteTask("wiki", queued[i].ID))
}
reserveSlots(t, svc, "kb-1", 8, 32)
for completed := 0; completed < 100; completed++ {
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending -= 10
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
worker := completed % len(runningIDs)
require.NoError(t, svc.redisClient.ZRem(ctx, wikiFollowUpPrefix+"kb-1", runningIDs[worker]).Err())
queued, err = inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
require.NoError(t, err)
require.NotEmpty(t, queued)
runningIDs[worker] = queued[0].ID
require.NoError(t, inspector.DeleteTask("wiki", queued[0].ID))
info, err := inspector.GetQueueInfo("wiki")
require.NoError(t, err)
require.LessOrEqual(t, info.Scheduled, 25, "completion %d must not multiply the queue", completed)
}
}
func TestScheduleFollowUpConcurrentCompletionsShareBudget(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
reserveSlots(t, svc, "kb-1", 16, 32)
start := make(chan struct{})
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
wikiFollowUpDelay, 10, 32, false)
}()
}
close(start)
wg.Wait()
info, err := inspector.GetQueueInfo("wiki")
require.NoError(t, err)
// Cap + one successor spare + at most one coalesced recovery probe.
require.LessOrEqual(t, info.Scheduled, 34)
require.Equal(t, int64(33), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
}
func TestScheduleFollowUpClaimedRowsOnlyKeepsCrashRecovery(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
zero := int64(0)
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).claimable = &zero
require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
wikiFollowUpDelay, 10, 32, false))
tasks, err := inspector.ListScheduledTasks("wiki")
require.NoError(t, err)
require.Len(t, tasks, 1)
require.Equal(t, "wiki-ingest-recheck-kb-1", tasks[0].ID)
require.Greater(t, time.Until(tasks[0].NextProcessAt), wikiClaimStaleAfter-time.Second)
}
func TestFollowUpReservationsKeepLongQueuedTasks(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
ctx := context.Background()
payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
for i := 0; i < 5; i++ {
expireFollowUpChecks(t, svc)
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
}
info, err := inspector.GetQueueInfo("wiki")
require.NoError(t, err)
require.LessOrEqual(t, info.Scheduled, 6) // five reserved tasks plus one probe
require.Equal(t, int64(5), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
}
func TestFollowUpReservationsRecoverMissingAndArchivedTasks(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
ctx := context.Background()
require.True(t, svc.scheduleFollowUp(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"},
wikiFollowUpDelay, 10, 4, false))
tasks, err := inspector.ListScheduledTasks("wiki")
require.NoError(t, err)
require.NoError(t, inspector.DeleteTask("wiki", tasks[0].ID))
require.NoError(t, inspector.ArchiveTask("wiki", tasks[1].ID))
expireFollowUpChecks(t, svc)
require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
require.NoError(t, err)
require.NotContains(t, ids, tasks[0].ID)
require.NotContains(t, ids, tasks[1].ID)
require.Len(t, ids, len(tasks)-2)
}
func TestFollowUpReservationsProtectEnqueueInProgress(t *testing.T) {
svc, _ := newQueuedFollowUpTestService(t)
ctx := context.Background()
ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
require.NoError(t, err)
require.Len(t, ids, 5)
// The task does not exist yet. Another completer must not recycle its
// capacity while the first enqueue is still in flight.
more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
require.NoError(t, err)
require.Empty(t, more)
// A producer that died before enqueueing must not consume capacity forever.
expireFollowUpChecks(t, svc)
more, err = svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
require.NoError(t, err)
require.Len(t, more, 5)
}
// An enqueue can be committed by Redis even when its response is lost.
type lostFollowUpEnqueueResponse struct{ client *asynq.Client }
func (q lostFollowUpEnqueueResponse) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
if _, err := q.client.Enqueue(task, opts...); err != nil {
return nil, err
}
return nil, errors.New("enqueue response lost")
}
func TestFollowUpReservationsSurviveAmbiguousEnqueueFailure(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
client := svc.task.(*asynq.Client)
svc.task = lostFollowUpEnqueueResponse{client: client}
svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
wikiFollowUpDelay, 10, 4, false)
svc.task = client
expireFollowUpChecks(t, svc)
require.NoError(t, svc.reconcileFollowUpTasks(context.Background(), "kb-1"))
require.Equal(t, int64(5), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
info, err := inspector.GetQueueInfo("wiki")
require.NoError(t, err)
require.Equal(t, 6, info.Scheduled) // five real tasks and the recovery probe
}
func TestFollowUpReservationHandlerLifecycle(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprintf("retry=%t", fail), func(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
kbService := &wikiGuardKBService{}
if fail {
kbService.err = errors.New("temporary KB lookup failure")
}
svc.kbService = kbService
ctx := context.Background()
ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 1, 1)
require.NoError(t, err)
require.Len(t, ids, 1)
server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
Concurrency: 1, Queues: map[string]int{"wiki": 1},
TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
RetryDelayFunc: func(int, error, *asynq.Task) time.Duration { return time.Hour },
LogLevel: asynq.ErrorLevel,
})
require.NoError(t, server.Start(asynq.HandlerFunc(svc.ProcessWikiIngest)))
t.Cleanup(server.Shutdown)
payload, err := json.Marshal(WikiIngestPayload{KnowledgeBaseID: "kb-1"})
require.NoError(t, err)
_, err = svc.task.Enqueue(asynq.NewTask("wiki:ingest", payload,
asynq.Queue("wiki"), asynq.TaskID(ids[0])))
require.NoError(t, err)
if fail {
require.Eventually(t, func() bool {
info, err := inspector.GetTaskInfo("wiki", ids[0])
return err == nil && info.State == asynq.TaskStateRetry
}, 5*time.Second, 10*time.Millisecond)
expireFollowUpChecks(t, svc)
require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
require.Equal(t, int64(1), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
} else {
require.Eventually(t, func() bool {
return svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val() == 0
}, 5*time.Second, 10*time.Millisecond, "successful handler must release its reservation")
}
})
}
}
func TestFollowUpRecheckCanRearmItself(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
Concurrency: 1, Queues: map[string]int{"wiki": 1},
TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
LogLevel: asynq.ErrorLevel,
})
require.NoError(t, server.Start(asynq.HandlerFunc(func(ctx context.Context, _ *asynq.Task) error {
if !svc.scheduleFollowUpRecheck(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}) {
return errors.New("could not re-arm probe")
}
return nil
})))
t.Cleanup(server.Shutdown)
_, err := svc.task.Enqueue(asynq.NewTask("wiki:ingest", nil,
asynq.Queue("wiki"), asynq.TaskID("wiki-followup-recheck-kb-1")))
require.NoError(t, err)
require.Eventually(t, func() bool {
info, err := inspector.GetTaskInfo("wiki", "wiki-followup-recheck-kb-1-next")
return err == nil && info.State == asynq.TaskStateScheduled
}, 5*time.Second, 10*time.Millisecond)
}
func TestFollowUpReservationsKeepCapacitySeparateForEachKB(t *testing.T) {
svc, _ := newQueuedFollowUpTestService(t)
ctx := context.Background()
first, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
require.NoError(t, err)
require.Len(t, first, 5)
second, err := svc.reserveFollowUpTasks(ctx, "kb-2", 5, 4)
require.NoError(t, err)
require.Len(t, second, 5)
more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
require.NoError(t, err)
require.Empty(t, more)
}
func TestScheduleFollowUpUsesRepositoryBatchLimit(t *testing.T) {
svc, inspector := newQueuedFollowUpTestService(t)
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending = 1500
reserveSlots(t, svc, "kb-1", 1, 4)
require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
wikiFollowUpDelay, 2000, 4, false))
info, err := inspector.GetQueueInfo("wiki")
require.NoError(t, err)
require.Equal(t, 2, info.Scheduled, "ClaimBatch can claim at most 1000 documents per batch")
}

View File

@ -86,6 +86,13 @@ type TaskPendingOpsRepository interface {
DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error
}
// TaskPendingOpsClaimableCounter counts the distinct keys ClaimBatch can
// claim, excluding every key with a fresh claim (including late siblings).
// PendingCount remains the count of all rows for crash-recovery consumers.
type TaskPendingOpsClaimableCounter interface {
ClaimableCount(ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time) (int64, error)
}
// TaskPendingOpsScopeCleaner is an optional extension for callers that need
// to discard every durable pending operation owned by a deleted scope. It is
// intentionally separate from TaskPendingOpsRepository so alternate queue