From 4ba85e2edf71a041fa6846c14eb74ccfa0cfb389 Mon Sep 17 00:00:00 2001 From: Duke Date: Sat, 26 Sep 2026 10:38:21 +0800 Subject: [PATCH] fix(wiki): fan out follow-up triggers so backlogs drain at ingest_max_inflight Since Phase 3 the consumption side (disjoint claiming, per-slug locks, the inflight cap) fully supports concurrent batches, but nothing created more than one trigger per completed batch: once uploads stopped, a KB with a standing backlog drained at 1 batch x batch-size per cycle regardless of ingest_max_inflight, leaving the wiki pool and the model server idle. followUpTriggerCount now re-arms min(batches the remaining rows still need, slots that could be free when the triggers fire). The caller's own slot releases right after scheduleFollowUp returns, hence the +1. activeInflightSlots is a read-mostly Lua companion to the reserve script (purge expired, then ZCARD); -1 means unknown (no Redis / Redis error) and collapses to a single follow-up, as do maxInflight <= 1 and Lite mode (liteLocks serialize per KB anyway). Fan-out is deliberately skipped on a rate-limited exit so a 429 storm is not amplified. Over-issued triggers are harmless: the ones that cannot get a slot are acked and coalesced by scheduleCappedRetry's fixed TaskID. The slot count renders as "unknown" in the log when it was not consulted. --- internal/application/service/wiki_ingest.go | 28 ++ .../application/service/wiki_ingest_batch.go | 107 +++++-- .../service/wiki_ingest_followup_test.go | 294 ++++++++++++++++++ 3 files changed, 407 insertions(+), 22 deletions(-) create mode 100644 internal/application/service/wiki_ingest_followup_test.go diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 4df7d57f..17203302 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -1112,6 +1112,34 @@ func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string }, true } +// wikiInflightCountScript purges expired slots exactly like the reserve +// script does, then returns the live count. Read-mostly companion to +// wikiInflightReserveScript so a fan-out decision sees the same view of the +// cap a reserver would. +const wikiInflightCountScript = ` +redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, tonumber(ARGV[1])) +return redis.call('ZCARD', KEYS[1]) +` + +// activeInflightSlots reports how many batches currently hold one of KB's +// in-flight slots (standard/Redis mode). Returns -1 when the count is +// unavailable (no Redis client, or a Redis error); callers must treat that as +// "unknown" and stay conservative instead of scaling on a stale number. +func (s *wikiIngestService) activeInflightSlots(ctx context.Context, kbID string) int { + if s.redisClient == nil { + return -1 + } + res, err := s.redisClient.Eval(ctx, wikiInflightCountScript, + []string{wikiInflightPrefix + kbID}, + time.Now().UnixMilli(), + ).Int() + if err != nil { + logger.Warnf(ctx, "wiki ingest: inflight count failed for KB %s: %v", kbID, err) + return -1 + } + return res +} + // scheduleCappedRetry enqueues a single coalesced follow-up trigger after a // batch was turned away by the in-flight cap. asynq.TaskID collapses all // turned-away triggers for one KB into a single pending retry (no thundering diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index c16c1758..796f7c0a 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -23,23 +23,68 @@ import ( "golang.org/x/sync/errgroup" ) -// scheduleFollowUp enqueues another asynq trigger task if there are -// still pending ops in task_pending_ops for this KB. Returns true when -// a follow-up was scheduled. +// followUpTriggerCount decides how many follow-up triggers a completed batch +// should re-arm for its KB (#3461). Since Phase 3 the consumption side +// (disjoint claiming, per-slug locks, the inflight cap) fully supports +// concurrent batches, but nothing created more than one trigger for a KB with +// a standing backlog — so once uploads stopped, the backlog drained at +// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving +// the wiki pool and the model server idle. // -// Post-Phase-3 this only backstops the case where a batch drained its -// claimed window but more rows remain and no other trigger is pending -// (e.g. steady trickle of uploads). Standard mode already fans a KB's -// backlog across concurrent claiming batches, so the short delay is -// normally just a light debounce rather than a lock-release wait. +// The count is the smaller of (a) how many batches the remaining rows still +// need and (b) how many slots could be free by the time the triggers fire: +// the caller's own slot releases right after it returns, hence the +1. +// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight +// <= 1 cannot host more than one batch anyway; both collapse to 1, matching +// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that +// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID. +func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int { + if pending <= 0 || batchSize <= 0 { + return 0 + } + batchesNeeded := (pending + batchSize - 1) / batchSize + if activeSlots < 0 || maxInflight <= 1 { + return 1 + } + freeSlots := maxInflight - activeSlots + 1 // caller's slot releases on return + if freeSlots < 1 { + freeSlots = 1 + } + return min(batchesNeeded, freeSlots) +} + +// scheduleFollowUp enqueues asynq trigger tasks if there are still pending +// ops in task_pending_ops for this KB. Returns true when at least one +// follow-up was scheduled. // -// `delay` is the ProcessIn before the follow-up fires. Callers pass +// Post-Phase-3 this backstops the case where a batch drained its claimed +// window but more rows remain and no other trigger is pending (e.g. a steady +// trickle of uploads). Standard mode already fans a KB's backlog across +// concurrent claiming batches, so the short delay is normally just a light +// debounce rather than a lock-release wait. +// +// How many triggers are re-armed follows followUpTriggerCount above: enough +// to drain the backlog, capped by the free inflight slots, so the concurrent +// batch count converges on ingest_max_inflight instead of idling at 1. +// +// Fan-out is deliberately skipped when rateLimited: widening concurrency +// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also +// never fans out — liteLocks serializes batches per KB, so extra triggers +// would only bounce off ErrWikiIngestConcurrent. +// +// `delay` is the ProcessIn before each follow-up fires. Callers pass // wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the // batch tripped an upstream rate limit — the released failed rows are // eligible immediately, but nothing claims them until a trigger fires, so // stretching the follow-up interval is what actually paces retries down // during a 429 storm. -func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool { +func (s *wikiIngestService) scheduleFollowUp( + ctx context.Context, + payload WikiIngestPayload, + delay time.Duration, + batchSize, maxInflight int, + rateLimited bool, +) bool { if s.pendingRepo == nil { return false } @@ -48,21 +93,39 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn return false } - logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay) + // activeSlots stays -1 (unknown → single follow-up) in Lite mode and on + // a rate-limited exit; -1 also shows up in the log line as "unknown". + activeSlots := -1 + if !rateLimited { + activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID) + } + n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight) + + activeStr := fmt.Sprintf("%d", activeSlots) + if activeSlots < 0 { + activeStr = "unknown" + } + logger.Infof(ctx, + "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)", + count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight) langfuse.InjectTracing(ctx, &payload) payloadBytes, _ := json.Marshal(payload) - t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, - asynq.Queue(types.QueueWiki), - asynq.MaxRetry(wikiIngestMaxRetry), - asynq.Timeout(60*time.Minute), - asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder - ) - if _, err := s.task.Enqueue(t); err != nil { - logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err) - return false + scheduled := false + for i := 0; i < n; i++ { + t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(60*time.Minute), + asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder + ) + if _, err := s.task.Enqueue(t); err != nil { + logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err) + continue + } + scheduled = true } - return true + return scheduled } // newWikiBatchContext builds the per-run lazy fetchers used by both the ingest @@ -946,7 +1009,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay) } followCtx, followCancel := wikiIngestCleanupContext(ctx) - followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay) + followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay, batchSize, maxInflight, rateLimited) followCancel() return nil } diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.go new file mode 100644 index 00000000..2536661e --- /dev/null +++ b/internal/application/service/wiki_ingest_followup_test.go @@ -0,0 +1,294 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +// --- test doubles ----------------------------------------------------------- + +// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes +// PendingCount configurable, which is the only input scheduleFollowUp reads. +type wikiPendingRepoForFollowUpTest struct { + wikiPendingRepoForCleanupTest + pending int64 +} + +func (r *wikiPendingRepoForFollowUpTest) PendingCount( + context.Context, string, string, string, +) (int64, error) { + return r.pending, nil +} + +// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues. +// failFrom >= 0 makes enqueues from that index onward return failErr so the +// partial-failure path can be exercised. +type followUpEnqueuerRecorder struct { + mu sync.Mutex + tasks []*asynq.Task + failFrom int + failErr error +} + +func (e *followUpEnqueuerRecorder) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + e.mu.Lock() + defer e.mu.Unlock() + idx := len(e.tasks) + e.tasks = append(e.tasks, task) + if e.failFrom >= 0 && idx >= e.failFrom { + return nil, e.failErr + } + return &asynq.TaskInfo{}, nil +} + +func (e *followUpEnqueuerRecorder) count() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.tasks) +} + +func (e *followUpEnqueuerRecorder) payloads(t *testing.T, kbID string) { + t.Helper() + e.mu.Lock() + defer e.mu.Unlock() + for _, task := range e.tasks { + var payload WikiIngestPayload + require.NoError(t, json.Unmarshal(task.Payload(), &payload)) + require.Equal(t, kbID, payload.KnowledgeBaseID) + } +} + +// --- pure fan-out computation ---------------------------------------------- + +func TestFollowUpTriggerCount(t *testing.T) { + tests := []struct { + name string + pending, batchSize, active, max int + want int + }{ + {"no pending enqueues nothing", 0, 5, 1, 4, 0}, + {"defensive zero batch size", 10, 0, 1, 4, 0}, + {"one batch or less stays at one", 5, 5, 1, 4, 1}, + {"single row stays at one", 1, 5, 1, 4, 1}, + {"just over one batch needs two", 6, 5, 1, 4, 2}, + {"backlog need is capped by free slots", 1000, 5, 2, 4, 3}, + {"all slots free counts own release", 1000, 5, 1, 4, 4}, + {"full board degrades to one", 1000, 5, 4, 4, 1}, + {"shrunk cap mid-flight clamps to one", 1000, 5, 7, 4, 1}, + {"unknown slot count is conservative", 1000, 5, -1, 4, 1}, + {"single-slot cap never fans out", 1000, 5, 0, 1, 1}, + {"exact multiple of batch size", 15, 5, 1, 4, 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := followUpTriggerCount(tt.pending, tt.batchSize, tt.active, tt.max) + require.Equal(t, tt.want, got) + }) + } +} + +// --- activeInflightSlots ---------------------------------------------------- + +func TestActiveInflightSlotsPurgesExpiredAndCountsLive(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + svc := &wikiIngestService{redisClient: rdb} + ctx := context.Background() + key := wikiInflightPrefix + "kb-1" + + // Two slots that expired long ago plus one live slot. + require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{ + Score: float64(time.Now().Add(-time.Hour).UnixMilli()), + Member: "expired-1", + }).Err()) + require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{ + Score: float64(time.Now().Add(-time.Minute).UnixMilli()), + Member: "expired-2", + }).Err()) + require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{ + Score: float64(time.Now().Add(wikiInflightTTL).UnixMilli()), + Member: "live-1", + }).Err()) + + require.Equal(t, 1, svc.activeInflightSlots(ctx, "kb-1")) +} + +func TestActiveInflightSlotsUnknownOnRedisError(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + require.NoError(t, rdb.Close()) // every Eval now fails + + svc := &wikiIngestService{redisClient: rdb} + require.Equal(t, -1, svc.activeInflightSlots(context.Background(), "kb-1")) +} + +// --- scheduleFollowUp ------------------------------------------------------- + +func newFollowUpTestService(t *testing.T, pending int64) (*wikiIngestService, *followUpEnqueuerRecorder) { + t.Helper() + rec := &followUpEnqueuerRecorder{failFrom: -1} + svc := &wikiIngestService{ + task: rec, + pendingRepo: &wikiPendingRepoForFollowUpTest{pending: pending}, + } + return svc, rec +} + +// reserveSlots claims n live slots the way n concurrent batches would, so the +// fan-out computation sees a realistic board. +func reserveSlots(t *testing.T, svc *wikiIngestService, kbID string, n, maxInflight int) { + t.Helper() + for i := 0; i < n; i++ { + release, granted := svc.reserveInflightSlot(context.Background(), kbID, maxInflight) + require.True(t, granted) + t.Cleanup(release) + } +} + +func TestScheduleFollowUpFansOutToFreeSlots(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + // 100 pending docs, batch size 10 → 10 batches needed. Two slots held + // (one of them modelling the caller's own, which releases on return), + // cap 4 → free = 4-2+1 = 3 follow-ups. + svc, rec := newFollowUpTestService(t, 100) + svc.redisClient = rdb + reserveSlots(t, svc, "kb-1", 2, 4) + + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false) + require.True(t, got) + require.Equal(t, 3, rec.count()) + rec.payloads(t, "kb-1") +} + +func TestScheduleFollowUpBacklogSmallerThanBatchStaysSingle(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + svc, rec := newFollowUpTestService(t, 7) + svc.redisClient = rdb + reserveSlots(t, svc, "kb-1", 1, 4) + + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false) + require.True(t, got) + require.Equal(t, 1, rec.count()) +} + +func TestScheduleFollowUpRateLimitedStaysSingle(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + // Even with the whole board free and a huge backlog, a rate-limited exit + // must not widen concurrency: widening during a 429 storm amplifies it. + svc, rec := newFollowUpTestService(t, 1000) + svc.redisClient = rdb + + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiRateLimitBackoff, 10, 32, true) + require.True(t, got) + require.Equal(t, 1, rec.count()) +} + +func TestScheduleFollowUpLiteModeStaysSingle(t *testing.T) { + // No Redis client → liteLocks serializes batches per KB; extra triggers + // would only bounce off ErrWikiIngestConcurrent. + svc, rec := newFollowUpTestService(t, 1000) + + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 32, false) + require.True(t, got) + require.Equal(t, 1, rec.count()) +} + +func TestScheduleFollowUpNoPendingSchedulesNothing(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + svc, rec := newFollowUpTestService(t, 0) + svc.redisClient = rdb + + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false) + require.False(t, got) + require.Equal(t, 0, rec.count()) +} + +func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + svc, rec := newFollowUpTestService(t, 100) + svc.redisClient = rdb + rec.failFrom = 1 + rec.failErr = errors.New("enqueue failed") + + // The caller holds its own slot (releases on return): free = 4-1+1 = 4. + // The first enqueue succeeds, the rest fail; "scheduled" must still be + // true because at least one trigger got out. + reserveSlots(t, svc, "kb-1", 1, 4) + got := svc.scheduleFollowUp(context.Background(), + WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false) + require.True(t, got) + require.Equal(t, 4, rec.count()) +} + +// --- convergence ------------------------------------------------------------ + +// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions +// fanning out on a board whose slots never free (worst case for over-issue) +// and asserts the per-completion trigger count never exceeds the free slots, +// i.e. the supply side alone cannot push concurrency past ingest_max_inflight. +func TestFollowUpFanOutConvergesUnderCap(t *testing.T) { + const ( + maxInflight = 8 + batchSize = 10 + ) + var ( + pending = 1000 + activeSlots = 1 // the completing batch itself; releases on return + ) + seen := map[int]bool{} + for round := 0; round < 6; round++ { + n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight) + require.Greater(t, n, 0) + require.LessOrEqual(t, n, maxInflight-activeSlots+1, + "round %d: supply must not exceed free slots", round) + if activeSlots == maxInflight { + require.Equal(t, 1, n, + "round %d: a full board must clamp supply to the follow-up chain", round) + } + seen[n] = true + + // The n triggers fire: n-1 of them claim slots (one bounces off the + // cap into scheduleCappedRetry when the board is full), each claims a + // batch of batchSize rows. + newBatches := min(n, maxInflight-activeSlots) + pending -= newBatches * batchSize + activeSlots = maxInflight + if pending <= 0 { + pending = 1 // keep the loop meaningful; count stays capped below + } + } + // The first round must have fanned out beyond the follow-up chain — + // otherwise this would not be testing convergence at all. + require.Greater(t, len(seen), 1) +}