fix(wiki): bound outstanding follow-up tasks
parent
4ba85e2edf
commit
c819abd958
|
|
@ -480,6 +480,25 @@ func (r *taskPendingOpsRepository) PendingCount(
|
|||
return n, nil
|
||||
}
|
||||
|
||||
// ClaimableCount uses ClaimBatch's eligibility and grouping rules so running
|
||||
// documents and multiple queued operations for one document do not inflate
|
||||
// the number of batches a scheduler needs to dispatch.
|
||||
func (r *taskPendingOpsRepository) ClaimableCount(
|
||||
ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time,
|
||||
) (int64, error) {
|
||||
freshKeys := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
|
||||
Select("dedup_key").
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
|
||||
var n int64
|
||||
err := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("claimed_at IS NULL OR claimed_at < ?", staleBefore).
|
||||
Where("dedup_key NOT IN (?)", freshKeys).
|
||||
Distinct("dedup_key").Count(&n).Error
|
||||
return n, err
|
||||
}
|
||||
|
||||
// DeleteByDedupKey drops rows in the tuple whose dedup_key matches.
|
||||
// If `op` is non-empty, only rows with the matching op are dropped;
|
||||
// otherwise every matching row is removed. Empty dedup_key is rejected
|
||||
|
|
|
|||
|
|
@ -1037,3 +1037,44 @@ func TestTaskPendingOps_DrainUnclaimedAndReleaseRollsBackOnReleaseFailure(t *tes
|
|||
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
func TestTaskPendingOps_ClaimableCountMatchesClaimBatch(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
counter := repo.(interfaces.TaskPendingOpsClaimableCounter)
|
||||
ctx := context.Background()
|
||||
staleBefore := time.Now().Add(-time.Hour)
|
||||
for _, key := range []string{"ready", "ready", "stale", "stale", "busy", "busy"} {
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", key, nil)))
|
||||
}
|
||||
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "stale").
|
||||
Update("claimed_at", staleBefore.Add(-time.Second)).Error)
|
||||
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "busy").
|
||||
Update("claimed_at", time.Now()).Error)
|
||||
// An unclaimed late sibling of a busy key is not eligible on its own.
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "retract", "busy", nil)))
|
||||
// Identical keys in another tuple must not exclude the ready document.
|
||||
other := makePendingOp("wiki:ingest", "knowledge_base", "other-kb", "ingest", "ready", nil)
|
||||
require.NoError(t, repo.Enqueue(ctx, other))
|
||||
require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("scope_id = ?", "other-kb").
|
||||
Update("claimed_at", time.Now()).Error)
|
||||
n, err := counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(2), n)
|
||||
rows, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb-1", 2, staleBefore)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 4)
|
||||
keys := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
keys[row.DedupKey] = true
|
||||
}
|
||||
require.Len(t, keys, int(n))
|
||||
n, err = counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, n)
|
||||
pending, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-1")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(7), pending, "crash recovery must still see claimed rows")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,26 +18,16 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// followUpTriggerCount decides how many follow-up triggers a completed batch
|
||||
// should re-arm for its KB (#3461). Since Phase 3 the consumption side
|
||||
// (disjoint claiming, per-slug locks, the inflight cap) fully supports
|
||||
// concurrent batches, but nothing created more than one trigger for a KB with
|
||||
// a standing backlog — so once uploads stopped, the backlog drained at
|
||||
// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
|
||||
// the wiki pool and the model server idle.
|
||||
//
|
||||
// The count is the smaller of (a) how many batches the remaining rows still
|
||||
// need and (b) how many slots could be free by the time the triggers fire:
|
||||
// the caller's own slot releases right after it returns, hence the +1.
|
||||
// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
|
||||
// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
|
||||
// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
|
||||
// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
|
||||
// followUpTriggerCount estimates useful new work from claimable documents and
|
||||
// running slots. reserveFollowUpTasks additionally bounds all outstanding
|
||||
// follow-ups (scheduled, queued, running and retrying) across batch completions.
|
||||
// The +1 allows the completing batch to replace itself before releasing its slot.
|
||||
func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
|
||||
if pending <= 0 || batchSize <= 0 {
|
||||
return 0
|
||||
|
|
@ -53,31 +43,9 @@ func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int
|
|||
return min(batchesNeeded, freeSlots)
|
||||
}
|
||||
|
||||
// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
|
||||
// ops in task_pending_ops for this KB. Returns true when at least one
|
||||
// follow-up was scheduled.
|
||||
//
|
||||
// Post-Phase-3 this backstops the case where a batch drained its claimed
|
||||
// window but more rows remain and no other trigger is pending (e.g. a steady
|
||||
// trickle of uploads). Standard mode already fans a KB's backlog across
|
||||
// concurrent claiming batches, so the short delay is normally just a light
|
||||
// debounce rather than a lock-release wait.
|
||||
//
|
||||
// How many triggers are re-armed follows followUpTriggerCount above: enough
|
||||
// to drain the backlog, capped by the free inflight slots, so the concurrent
|
||||
// batch count converges on ingest_max_inflight instead of idling at 1.
|
||||
//
|
||||
// Fan-out is deliberately skipped when rateLimited: widening concurrency
|
||||
// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
|
||||
// never fans out — liteLocks serializes batches per KB, so extra triggers
|
||||
// would only bounce off ErrWikiIngestConcurrent.
|
||||
//
|
||||
// `delay` is the ProcessIn before each follow-up fires. Callers pass
|
||||
// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
|
||||
// batch tripped an upstream rate limit — the released failed rows are
|
||||
// eligible immediately, but nothing claims them until a trigger fires, so
|
||||
// stretching the follow-up interval is what actually paces retries down
|
||||
// during a 429 storm.
|
||||
// scheduleFollowUp replenishes the KB's bounded supply of follow-up tasks.
|
||||
// Lite mode and rate-limited completions request only one successor. A 429
|
||||
// delays this batch's successor; it does not pause other batches in the KB.
|
||||
func (s *wikiIngestService) scheduleFollowUp(
|
||||
ctx context.Context,
|
||||
payload WikiIngestPayload,
|
||||
|
|
@ -93,38 +61,74 @@ func (s *wikiIngestService) scheduleFollowUp(
|
|||
return false
|
||||
}
|
||||
|
||||
if s.redisClient != nil {
|
||||
if counter, ok := s.pendingRepo.(interfaces.TaskPendingOpsClaimableCounter); ok {
|
||||
count, err = counter.ClaimableCount(ctx, wikiTaskType, wikiTaskScope,
|
||||
payload.KnowledgeBaseID, time.Now().Add(-wikiClaimStaleAfter))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: claimable count failed: %v", err)
|
||||
return s.scheduleFollowUpRecheck(ctx, payload)
|
||||
}
|
||||
if count == 0 {
|
||||
return s.scheduleStaleClaimRecheck(ctx, payload)
|
||||
}
|
||||
}
|
||||
// ClaimBatch caps the number of distinct keys at 1000.
|
||||
batchSize = min(batchSize, 1000)
|
||||
}
|
||||
|
||||
// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
|
||||
// a rate-limited exit; -1 also shows up in the log line as "unknown".
|
||||
// a rate-limited exit.
|
||||
activeSlots := -1
|
||||
if !rateLimited {
|
||||
activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
|
||||
}
|
||||
n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
|
||||
|
||||
activeStr := fmt.Sprintf("%d", activeSlots)
|
||||
if activeSlots < 0 {
|
||||
activeStr = "unknown"
|
||||
var taskIDs []string
|
||||
if s.redisClient != nil {
|
||||
taskIDs, err = s.reserveFollowUpTasks(ctx, payload.KnowledgeBaseID, n, maxInflight)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up reservation failed: %v", err)
|
||||
// Keep the pre-fan-out behavior while Redis is unavailable.
|
||||
n = 1
|
||||
} else {
|
||||
n = len(taskIDs)
|
||||
if n == 0 {
|
||||
return s.scheduleFollowUpRecheck(ctx, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx,
|
||||
"wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
|
||||
count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
|
||||
logger.Infof(ctx, "wiki ingest: %d claimable documents for KB %s, scheduling %d follow-up(s) in %s",
|
||||
count, payload.KnowledgeBaseID, n, delay)
|
||||
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
payloadBytes, _ := json.Marshal(payload)
|
||||
scheduled := false
|
||||
needsRecheck := false
|
||||
for i := 0; i < n; i++ {
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
opts := []asynq.Option{
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
|
||||
)
|
||||
asynq.Timeout(60 * time.Minute),
|
||||
asynq.ProcessIn(delay),
|
||||
}
|
||||
if len(taskIDs) > 0 {
|
||||
opts = append(opts, asynq.TaskID(taskIDs[i]))
|
||||
}
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, opts...)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
|
||||
// A timeout may mean the enqueue succeeded. Keep its reservation
|
||||
// until reconciliation can establish the actual queue state.
|
||||
needsRecheck = true
|
||||
continue
|
||||
}
|
||||
scheduled = true
|
||||
}
|
||||
if needsRecheck {
|
||||
scheduled = s.scheduleFollowUpRecheck(ctx, payload) || scheduled
|
||||
}
|
||||
return scheduled
|
||||
}
|
||||
|
||||
|
|
@ -253,7 +257,7 @@ func (s *wikiIngestService) newWikiBatchContext(
|
|||
}
|
||||
}
|
||||
|
||||
func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {
|
||||
func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) (retErr error) {
|
||||
taskStartedAt := time.Now()
|
||||
retryCount, _ := asynq.GetRetryCount(ctx)
|
||||
maxRetry, _ := asynq.GetMaxRetry(ctx)
|
||||
|
|
@ -312,6 +316,16 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
|||
return fmt.Errorf("wiki ingest: unmarshal payload: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
// Asynq retries panics too: retain the outstanding reservation.
|
||||
panic(p)
|
||||
}
|
||||
if retErr == nil {
|
||||
s.releaseFollowUpTask(ctx, payload.KnowledgeBaseID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Inject context
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,152 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
wikiFollowUpPrefix = "wiki:followup:"
|
||||
// This is a reconciliation interval, not a queue lease. A task can wait
|
||||
// arbitrarily long for a worker without losing its reserved capacity.
|
||||
// Missing tasks are retained for this interval to allow an enqueue in
|
||||
// progress to finish, including when its caller times out.
|
||||
wikiFollowUpReconcileAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Reservations cover the entire asynq task lifetime, including delayed tasks
|
||||
// and retries. Keep one spare beyond the running cap so the last completing
|
||||
// task can enqueue its successor before its own reservation is released.
|
||||
// Atomic ZCARD + ZADD makes concurrent completions share the same budget.
|
||||
const wikiFollowUpReserveScript = `
|
||||
local available = math.max(0, tonumber(ARGV[2]) - redis.call('ZCARD', KEYS[1]))
|
||||
local n = math.min(tonumber(ARGV[1]), available)
|
||||
local ids = {}
|
||||
for i = 1, n do
|
||||
local id = ARGV[3] .. ':' .. i
|
||||
redis.call('ZADD', KEYS[1], ARGV[4], id)
|
||||
ids[#ids + 1] = id
|
||||
end
|
||||
return ids
|
||||
`
|
||||
|
||||
// Do not expire a reservation just because the task has waited a long time.
|
||||
// Inspect asynq first, then remove/refresh only if another reconciler hasn't
|
||||
// already updated it. ZADD XX also avoids resurrecting a completed task.
|
||||
const wikiFollowUpReconcileScript = `
|
||||
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
|
||||
if score and tonumber(score) == tonumber(ARGV[2]) then
|
||||
if ARGV[3] == 'remove' then
|
||||
redis.call('ZREM', KEYS[1], ARGV[1])
|
||||
else
|
||||
redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1])
|
||||
end
|
||||
end
|
||||
return 1
|
||||
`
|
||||
|
||||
func (s *wikiIngestService) reserveFollowUpTasks(
|
||||
ctx context.Context, kbID string, requested, maxInflight int,
|
||||
) ([]string, error) {
|
||||
if requested <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := s.reconcileFollowUpTasks(ctx, kbID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.redisClient.Eval(ctx, wikiFollowUpReserveScript,
|
||||
[]string{wikiFollowUpPrefix + kbID}, requested, max(1, maxInflight)+1,
|
||||
"wiki-followup-"+uuid.NewString(), time.Now().Add(wikiFollowUpReconcileAfter).UnixMilli(),
|
||||
).StringSlice()
|
||||
}
|
||||
|
||||
func (s *wikiIngestService) reconcileFollowUpTasks(ctx context.Context, kbID string) error {
|
||||
key := wikiFollowUpPrefix + kbID
|
||||
now := time.Now()
|
||||
entries, err := s.redisClient.ZRangeByScoreWithScores(ctx, key, &redis.ZRangeBy{
|
||||
Min: "-inf", Max: strconv.FormatInt(now.UnixMilli(), 10),
|
||||
}).Result()
|
||||
if err != nil || len(entries) == 0 {
|
||||
return err
|
||||
}
|
||||
// Shares the service's Redis connection; Inspector.Close must not close it.
|
||||
inspector := asynq.NewInspectorFromRedisClient(s.redisClient)
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
id, ok := entry.Member.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
info, err := inspector.GetTaskInfo(types.QueueWiki, id)
|
||||
missing := errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound)
|
||||
if err != nil && !missing {
|
||||
return err
|
||||
}
|
||||
action := strconv.FormatInt(now.Add(wikiFollowUpReconcileAfter).UnixMilli(), 10)
|
||||
if missing || info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {
|
||||
action = "remove"
|
||||
}
|
||||
if err := s.redisClient.Eval(ctx, wikiFollowUpReconcileScript,
|
||||
[]string{key}, id, entry.Score, action).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wikiIngestService) releaseFollowUpTask(ctx context.Context, kbID string) {
|
||||
if s.redisClient == nil {
|
||||
return
|
||||
}
|
||||
id, ok := asynq.GetTaskID(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
cleanupCtx, cancel := wikiIngestCleanupContext(ctx)
|
||||
defer cancel()
|
||||
if err := s.redisClient.ZRem(cleanupCtx, wikiFollowUpPrefix+kbID, id).Err(); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up reservation release failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A coalesced probe keeps progress possible when all capacity is reserved but
|
||||
// an enqueue failed or a worker died. It also revisits reservations whose
|
||||
// cleanup failed. It is outside the reservation budget and contributes at
|
||||
// most two probes per KB, independent of the backlog size.
|
||||
func (s *wikiIngestService) scheduleFollowUpRecheck(ctx context.Context, payload WikiIngestPayload) bool {
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
b, _ := json.Marshal(payload)
|
||||
id := "wiki-followup-recheck-" + payload.KnowledgeBaseID
|
||||
if current, _ := asynq.GetTaskID(ctx); current == id {
|
||||
// The current task ID remains occupied until asynq acknowledges it.
|
||||
// Alternate between two IDs so the probe can re-arm itself.
|
||||
id += "-next"
|
||||
}
|
||||
t := asynq.NewTask(types.TypeWikiIngest, b,
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(wikiFollowUpReconcileAfter+wikiFollowUpDelay),
|
||||
asynq.TaskID(id),
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
|
||||
return true
|
||||
}
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up recheck enqueue failed: %v", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -17,10 +18,12 @@ import (
|
|||
// --- test doubles -----------------------------------------------------------
|
||||
|
||||
// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
|
||||
// PendingCount configurable, which is the only input scheduleFollowUp reads.
|
||||
// PendingCount and the eligible-document count configurable.
|
||||
type wikiPendingRepoForFollowUpTest struct {
|
||||
wikiPendingRepoForCleanupTest
|
||||
pending int64
|
||||
pending int64
|
||||
claimable *int64
|
||||
claimableErr error
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForFollowUpTest) PendingCount(
|
||||
|
|
@ -29,6 +32,15 @@ func (r *wikiPendingRepoForFollowUpTest) PendingCount(
|
|||
return r.pending, nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForFollowUpTest) ClaimableCount(
|
||||
context.Context, string, string, string, time.Time,
|
||||
) (int64, error) {
|
||||
if r.claimable != nil {
|
||||
return *r.claimable, r.claimableErr
|
||||
}
|
||||
return r.pending, r.claimableErr
|
||||
}
|
||||
|
||||
// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
|
||||
// failFrom >= 0 makes enqueues from that index onward return failErr so the
|
||||
// partial-failure path can be exercised.
|
||||
|
|
@ -248,47 +260,274 @@ func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T
|
|||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 4, rec.count())
|
||||
require.Equal(t, 5, rec.count()) // four follow-ups and one recovery probe
|
||||
}
|
||||
|
||||
// --- convergence ------------------------------------------------------------
|
||||
// Use the actual asynq client so these tests exercise atomic reservations and
|
||||
// queue state, rather than merely checking the single-completer arithmetic.
|
||||
func newQueuedFollowUpTestService(t *testing.T) (*wikiIngestService, *asynq.Inspector) {
|
||||
t.Helper()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
client := asynq.NewClient(asynq.RedisClientOpt{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
return &wikiIngestService{
|
||||
redisClient: rdb,
|
||||
task: client,
|
||||
pendingRepo: &wikiPendingRepoForFollowUpTest{pending: 24624},
|
||||
}, asynq.NewInspectorFromRedisClient(rdb)
|
||||
}
|
||||
|
||||
// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
|
||||
// fanning out on a board whose slots never free (worst case for over-issue)
|
||||
// and asserts the per-completion trigger count never exceeds the free slots,
|
||||
// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
|
||||
func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
|
||||
const (
|
||||
maxInflight = 8
|
||||
batchSize = 10
|
||||
)
|
||||
var (
|
||||
pending = 1000
|
||||
activeSlots = 1 // the completing batch itself; releases on return
|
||||
)
|
||||
seen := map[int]bool{}
|
||||
for round := 0; round < 6; round++ {
|
||||
n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
|
||||
require.Greater(t, n, 0)
|
||||
require.LessOrEqual(t, n, maxInflight-activeSlots+1,
|
||||
"round %d: supply must not exceed free slots", round)
|
||||
if activeSlots == maxInflight {
|
||||
require.Equal(t, 1, n,
|
||||
"round %d: a full board must clamp supply to the follow-up chain", round)
|
||||
}
|
||||
seen[n] = true
|
||||
|
||||
// The n triggers fire: n-1 of them claim slots (one bounces off the
|
||||
// cap into scheduleCappedRetry when the board is full), each claims a
|
||||
// batch of batchSize rows.
|
||||
newBatches := min(n, maxInflight-activeSlots)
|
||||
pending -= newBatches * batchSize
|
||||
activeSlots = maxInflight
|
||||
if pending <= 0 {
|
||||
pending = 1 // keep the loop meaningful; count stays capped below
|
||||
}
|
||||
func expireFollowUpChecks(t *testing.T, svc *wikiIngestService) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
for _, id := range ids {
|
||||
require.NoError(t, svc.redisClient.ZAdd(ctx, wikiFollowUpPrefix+"kb-1", redis.Z{
|
||||
Score: float64(time.Now().Add(-time.Minute).UnixMilli()), Member: id,
|
||||
}).Err())
|
||||
}
|
||||
// The first round must have fanned out beyond the follow-up chain —
|
||||
// otherwise this would not be testing convergence at all.
|
||||
require.Greater(t, len(seen), 1)
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpWorkerPoolBelowCap(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
ctx := context.Background()
|
||||
payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
|
||||
// Startup recovery has one running batch, whose completion fans out.
|
||||
release, ok := svc.reserveInflightSlot(ctx, "kb-1", 32)
|
||||
require.True(t, ok)
|
||||
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
|
||||
release()
|
||||
queued, err := inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, queued, 32)
|
||||
|
||||
// Model eight workers dequeuing tasks. Keep their follow-up reservations
|
||||
// until completion, just as the handler does, and consume one queued task
|
||||
// for each finished batch. No model, database or external Redis is used.
|
||||
runningIDs := make([]string, 8)
|
||||
for i := range runningIDs {
|
||||
runningIDs[i] = queued[i].ID
|
||||
require.NoError(t, inspector.DeleteTask("wiki", queued[i].ID))
|
||||
}
|
||||
reserveSlots(t, svc, "kb-1", 8, 32)
|
||||
for completed := 0; completed < 100; completed++ {
|
||||
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending -= 10
|
||||
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
|
||||
worker := completed % len(runningIDs)
|
||||
require.NoError(t, svc.redisClient.ZRem(ctx, wikiFollowUpPrefix+"kb-1", runningIDs[worker]).Err())
|
||||
queued, err = inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, queued)
|
||||
runningIDs[worker] = queued[0].ID
|
||||
require.NoError(t, inspector.DeleteTask("wiki", queued[0].ID))
|
||||
info, err := inspector.GetQueueInfo("wiki")
|
||||
require.NoError(t, err)
|
||||
require.LessOrEqual(t, info.Scheduled, 25, "completion %d must not multiply the queue", completed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpConcurrentCompletionsShareBudget(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
reserveSlots(t, svc, "kb-1", 16, 32)
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
|
||||
wikiFollowUpDelay, 10, 32, false)
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
info, err := inspector.GetQueueInfo("wiki")
|
||||
require.NoError(t, err)
|
||||
// Cap + one successor spare + at most one coalesced recovery probe.
|
||||
require.LessOrEqual(t, info.Scheduled, 34)
|
||||
require.Equal(t, int64(33), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpClaimedRowsOnlyKeepsCrashRecovery(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
zero := int64(0)
|
||||
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).claimable = &zero
|
||||
require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
|
||||
wikiFollowUpDelay, 10, 32, false))
|
||||
tasks, err := inspector.ListScheduledTasks("wiki")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, tasks, 1)
|
||||
require.Equal(t, "wiki-ingest-recheck-kb-1", tasks[0].ID)
|
||||
require.Greater(t, time.Until(tasks[0].NextProcessAt), wikiClaimStaleAfter-time.Second)
|
||||
}
|
||||
|
||||
func TestFollowUpReservationsKeepLongQueuedTasks(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
ctx := context.Background()
|
||||
payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
|
||||
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
|
||||
for i := 0; i < 5; i++ {
|
||||
expireFollowUpChecks(t, svc)
|
||||
require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
|
||||
}
|
||||
info, err := inspector.GetQueueInfo("wiki")
|
||||
require.NoError(t, err)
|
||||
require.LessOrEqual(t, info.Scheduled, 6) // five reserved tasks plus one probe
|
||||
require.Equal(t, int64(5), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
|
||||
}
|
||||
|
||||
func TestFollowUpReservationsRecoverMissingAndArchivedTasks(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
ctx := context.Background()
|
||||
require.True(t, svc.scheduleFollowUp(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"},
|
||||
wikiFollowUpDelay, 10, 4, false))
|
||||
tasks, err := inspector.ListScheduledTasks("wiki")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, inspector.DeleteTask("wiki", tasks[0].ID))
|
||||
require.NoError(t, inspector.ArchiveTask("wiki", tasks[1].ID))
|
||||
expireFollowUpChecks(t, svc)
|
||||
require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
|
||||
ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, ids, tasks[0].ID)
|
||||
require.NotContains(t, ids, tasks[1].ID)
|
||||
require.Len(t, ids, len(tasks)-2)
|
||||
}
|
||||
|
||||
func TestFollowUpReservationsProtectEnqueueInProgress(t *testing.T) {
|
||||
svc, _ := newQueuedFollowUpTestService(t)
|
||||
ctx := context.Background()
|
||||
ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ids, 5)
|
||||
// The task does not exist yet. Another completer must not recycle its
|
||||
// capacity while the first enqueue is still in flight.
|
||||
more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, more)
|
||||
// A producer that died before enqueueing must not consume capacity forever.
|
||||
expireFollowUpChecks(t, svc)
|
||||
more, err = svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, more, 5)
|
||||
}
|
||||
|
||||
// An enqueue can be committed by Redis even when its response is lost.
|
||||
type lostFollowUpEnqueueResponse struct{ client *asynq.Client }
|
||||
|
||||
func (q lostFollowUpEnqueueResponse) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
if _, err := q.client.Enqueue(task, opts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.New("enqueue response lost")
|
||||
}
|
||||
|
||||
func TestFollowUpReservationsSurviveAmbiguousEnqueueFailure(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
client := svc.task.(*asynq.Client)
|
||||
svc.task = lostFollowUpEnqueueResponse{client: client}
|
||||
svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
|
||||
wikiFollowUpDelay, 10, 4, false)
|
||||
svc.task = client
|
||||
expireFollowUpChecks(t, svc)
|
||||
require.NoError(t, svc.reconcileFollowUpTasks(context.Background(), "kb-1"))
|
||||
require.Equal(t, int64(5), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
|
||||
info, err := inspector.GetQueueInfo("wiki")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 6, info.Scheduled) // five real tasks and the recovery probe
|
||||
}
|
||||
|
||||
func TestFollowUpReservationHandlerLifecycle(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("retry=%t", fail), func(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
kbService := &wikiGuardKBService{}
|
||||
if fail {
|
||||
kbService.err = errors.New("temporary KB lookup failure")
|
||||
}
|
||||
svc.kbService = kbService
|
||||
ctx := context.Background()
|
||||
ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 1, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ids, 1)
|
||||
server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
|
||||
Concurrency: 1, Queues: map[string]int{"wiki": 1},
|
||||
TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
|
||||
RetryDelayFunc: func(int, error, *asynq.Task) time.Duration { return time.Hour },
|
||||
LogLevel: asynq.ErrorLevel,
|
||||
})
|
||||
require.NoError(t, server.Start(asynq.HandlerFunc(svc.ProcessWikiIngest)))
|
||||
t.Cleanup(server.Shutdown)
|
||||
payload, err := json.Marshal(WikiIngestPayload{KnowledgeBaseID: "kb-1"})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.task.Enqueue(asynq.NewTask("wiki:ingest", payload,
|
||||
asynq.Queue("wiki"), asynq.TaskID(ids[0])))
|
||||
require.NoError(t, err)
|
||||
if fail {
|
||||
require.Eventually(t, func() bool {
|
||||
info, err := inspector.GetTaskInfo("wiki", ids[0])
|
||||
return err == nil && info.State == asynq.TaskStateRetry
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
expireFollowUpChecks(t, svc)
|
||||
require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
|
||||
require.Equal(t, int64(1), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
|
||||
} else {
|
||||
require.Eventually(t, func() bool {
|
||||
return svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val() == 0
|
||||
}, 5*time.Second, 10*time.Millisecond, "successful handler must release its reservation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFollowUpRecheckCanRearmItself(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
|
||||
Concurrency: 1, Queues: map[string]int{"wiki": 1},
|
||||
TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
|
||||
LogLevel: asynq.ErrorLevel,
|
||||
})
|
||||
require.NoError(t, server.Start(asynq.HandlerFunc(func(ctx context.Context, _ *asynq.Task) error {
|
||||
if !svc.scheduleFollowUpRecheck(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}) {
|
||||
return errors.New("could not re-arm probe")
|
||||
}
|
||||
return nil
|
||||
})))
|
||||
t.Cleanup(server.Shutdown)
|
||||
_, err := svc.task.Enqueue(asynq.NewTask("wiki:ingest", nil,
|
||||
asynq.Queue("wiki"), asynq.TaskID("wiki-followup-recheck-kb-1")))
|
||||
require.NoError(t, err)
|
||||
require.Eventually(t, func() bool {
|
||||
info, err := inspector.GetTaskInfo("wiki", "wiki-followup-recheck-kb-1-next")
|
||||
return err == nil && info.State == asynq.TaskStateScheduled
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestFollowUpReservationsKeepCapacitySeparateForEachKB(t *testing.T) {
|
||||
svc, _ := newQueuedFollowUpTestService(t)
|
||||
ctx := context.Background()
|
||||
first, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 5)
|
||||
second, err := svc.reserveFollowUpTasks(ctx, "kb-2", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, second, 5)
|
||||
more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, more)
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpUsesRepositoryBatchLimit(t *testing.T) {
|
||||
svc, inspector := newQueuedFollowUpTestService(t)
|
||||
svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending = 1500
|
||||
reserveSlots(t, svc, "kb-1", 1, 4)
|
||||
require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
|
||||
wikiFollowUpDelay, 2000, 4, false))
|
||||
info, err := inspector.GetQueueInfo("wiki")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, info.Scheduled, "ClaimBatch can claim at most 1000 documents per batch")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,13 @@ type TaskPendingOpsRepository interface {
|
|||
DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error
|
||||
}
|
||||
|
||||
// TaskPendingOpsClaimableCounter counts the distinct keys ClaimBatch can
|
||||
// claim, excluding every key with a fresh claim (including late siblings).
|
||||
// PendingCount remains the count of all rows for crash-recovery consumers.
|
||||
type TaskPendingOpsClaimableCounter interface {
|
||||
ClaimableCount(ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// TaskPendingOpsScopeCleaner is an optional extension for callers that need
|
||||
// to discard every durable pending operation owned by a deleted scope. It is
|
||||
// intentionally separate from TaskPendingOpsRepository so alternate queue
|
||||
|
|
|
|||
Loading…
Reference in New Issue