fix(wiki): fan out follow-up triggers so backlogs drain at ingest_max_inflight

Since Phase 3 the consumption side (disjoint claiming, per-slug locks,
the inflight cap) fully supports concurrent batches, but nothing created
more than one trigger per completed batch: once uploads stopped, a KB
with a standing backlog drained at 1 batch x batch-size per cycle
regardless of ingest_max_inflight, leaving the wiki pool and the model
server idle.

followUpTriggerCount now re-arms min(batches the remaining rows still
need, slots that could be free when the triggers fire). The caller's
own slot releases right after scheduleFollowUp returns, hence the +1.
activeInflightSlots is a read-mostly Lua companion to the reserve
script (purge expired, then ZCARD); -1 means unknown (no Redis / Redis
error) and collapses to a single follow-up, as do maxInflight <= 1 and
Lite mode (liteLocks serialize per KB anyway). Fan-out is deliberately
skipped on a rate-limited exit so a 429 storm is not amplified.

Over-issued triggers are harmless: the ones that cannot get a slot are
acked and coalesced by scheduleCappedRetry's fixed TaskID. The slot
count renders as "unknown" in the log when it was not consulted.
main
Duke 2026-09-26 10:38:21 +08:00 committed by lyingbug
parent bad7afa6ae
commit 4ba85e2edf
3 changed files with 407 additions and 22 deletions

View File

@ -1112,6 +1112,34 @@ func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string
}, true
}
// wikiInflightCountScript purges expired slots exactly like the reserve
// script does, then returns the live count. Read-mostly companion to
// wikiInflightReserveScript so a fan-out decision sees the same view of the
// cap a reserver would.
const wikiInflightCountScript = `
redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, tonumber(ARGV[1]))
return redis.call('ZCARD', KEYS[1])
`
// activeInflightSlots reports how many batches currently hold one of KB's
// in-flight slots (standard/Redis mode). Returns -1 when the count is
// unavailable (no Redis client, or a Redis error); callers must treat that as
// "unknown" and stay conservative instead of scaling on a stale number.
func (s *wikiIngestService) activeInflightSlots(ctx context.Context, kbID string) int {
if s.redisClient == nil {
return -1
}
res, err := s.redisClient.Eval(ctx, wikiInflightCountScript,
[]string{wikiInflightPrefix + kbID},
time.Now().UnixMilli(),
).Int()
if err != nil {
logger.Warnf(ctx, "wiki ingest: inflight count failed for KB %s: %v", kbID, err)
return -1
}
return res
}
// scheduleCappedRetry enqueues a single coalesced follow-up trigger after a
// batch was turned away by the in-flight cap. asynq.TaskID collapses all
// turned-away triggers for one KB into a single pending retry (no thundering

View File

@ -23,23 +23,68 @@ import (
"golang.org/x/sync/errgroup"
)
// scheduleFollowUp enqueues another asynq trigger task if there are
// still pending ops in task_pending_ops for this KB. Returns true when
// a follow-up was scheduled.
// followUpTriggerCount decides how many follow-up triggers a completed batch
// should re-arm for its KB (#3461). Since Phase 3 the consumption side
// (disjoint claiming, per-slug locks, the inflight cap) fully supports
// concurrent batches, but nothing created more than one trigger for a KB with
// a standing backlog — so once uploads stopped, the backlog drained at
// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
// the wiki pool and the model server idle.
//
// Post-Phase-3 this only backstops the case where a batch drained its
// claimed window but more rows remain and no other trigger is pending
// (e.g. steady trickle of uploads). Standard mode already fans a KB's
// backlog across concurrent claiming batches, so the short delay is
// normally just a light debounce rather than a lock-release wait.
// The count is the smaller of (a) how many batches the remaining rows still
// need and (b) how many slots could be free by the time the triggers fire:
// the caller's own slot releases right after it returns, hence the +1.
// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
if pending <= 0 || batchSize <= 0 {
return 0
}
batchesNeeded := (pending + batchSize - 1) / batchSize
if activeSlots < 0 || maxInflight <= 1 {
return 1
}
freeSlots := maxInflight - activeSlots + 1 // caller's slot releases on return
if freeSlots < 1 {
freeSlots = 1
}
return min(batchesNeeded, freeSlots)
}
// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
// ops in task_pending_ops for this KB. Returns true when at least one
// follow-up was scheduled.
//
// `delay` is the ProcessIn before the follow-up fires. Callers pass
// Post-Phase-3 this backstops the case where a batch drained its claimed
// window but more rows remain and no other trigger is pending (e.g. a steady
// trickle of uploads). Standard mode already fans a KB's backlog across
// concurrent claiming batches, so the short delay is normally just a light
// debounce rather than a lock-release wait.
//
// How many triggers are re-armed follows followUpTriggerCount above: enough
// to drain the backlog, capped by the free inflight slots, so the concurrent
// batch count converges on ingest_max_inflight instead of idling at 1.
//
// Fan-out is deliberately skipped when rateLimited: widening concurrency
// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
// never fans out — liteLocks serializes batches per KB, so extra triggers
// would only bounce off ErrWikiIngestConcurrent.
//
// `delay` is the ProcessIn before each follow-up fires. Callers pass
// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
// batch tripped an upstream rate limit — the released failed rows are
// eligible immediately, but nothing claims them until a trigger fires, so
// stretching the follow-up interval is what actually paces retries down
// during a 429 storm.
func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool {
func (s *wikiIngestService) scheduleFollowUp(
ctx context.Context,
payload WikiIngestPayload,
delay time.Duration,
batchSize, maxInflight int,
rateLimited bool,
) bool {
if s.pendingRepo == nil {
return false
}
@ -48,21 +93,39 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
return false
}
logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay)
// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
// a rate-limited exit; -1 also shows up in the log line as "unknown".
activeSlots := -1
if !rateLimited {
activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
}
n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
activeStr := fmt.Sprintf("%d", activeSlots)
if activeSlots < 0 {
activeStr = "unknown"
}
logger.Infof(ctx,
"wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
langfuse.InjectTracing(ctx, &payload)
payloadBytes, _ := json.Marshal(payload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
)
if _, err := s.task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)
return false
scheduled := false
for i := 0; i < n; i++ {
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
)
if _, err := s.task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
continue
}
scheduled = true
}
return true
return scheduled
}
// newWikiBatchContext builds the per-run lazy fetchers used by both the ingest
@ -946,7 +1009,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay)
}
followCtx, followCancel := wikiIngestCleanupContext(ctx)
followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay)
followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay, batchSize, maxInflight, rateLimited)
followCancel()
return nil
}

View File

@ -0,0 +1,294 @@
package service
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
// --- test doubles -----------------------------------------------------------
// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
// PendingCount configurable, which is the only input scheduleFollowUp reads.
type wikiPendingRepoForFollowUpTest struct {
wikiPendingRepoForCleanupTest
pending int64
}
func (r *wikiPendingRepoForFollowUpTest) PendingCount(
context.Context, string, string, string,
) (int64, error) {
return r.pending, nil
}
// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
// failFrom >= 0 makes enqueues from that index onward return failErr so the
// partial-failure path can be exercised.
type followUpEnqueuerRecorder struct {
mu sync.Mutex
tasks []*asynq.Task
failFrom int
failErr error
}
func (e *followUpEnqueuerRecorder) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
e.mu.Lock()
defer e.mu.Unlock()
idx := len(e.tasks)
e.tasks = append(e.tasks, task)
if e.failFrom >= 0 && idx >= e.failFrom {
return nil, e.failErr
}
return &asynq.TaskInfo{}, nil
}
func (e *followUpEnqueuerRecorder) count() int {
e.mu.Lock()
defer e.mu.Unlock()
return len(e.tasks)
}
func (e *followUpEnqueuerRecorder) payloads(t *testing.T, kbID string) {
t.Helper()
e.mu.Lock()
defer e.mu.Unlock()
for _, task := range e.tasks {
var payload WikiIngestPayload
require.NoError(t, json.Unmarshal(task.Payload(), &payload))
require.Equal(t, kbID, payload.KnowledgeBaseID)
}
}
// --- pure fan-out computation ----------------------------------------------
func TestFollowUpTriggerCount(t *testing.T) {
tests := []struct {
name string
pending, batchSize, active, max int
want int
}{
{"no pending enqueues nothing", 0, 5, 1, 4, 0},
{"defensive zero batch size", 10, 0, 1, 4, 0},
{"one batch or less stays at one", 5, 5, 1, 4, 1},
{"single row stays at one", 1, 5, 1, 4, 1},
{"just over one batch needs two", 6, 5, 1, 4, 2},
{"backlog need is capped by free slots", 1000, 5, 2, 4, 3},
{"all slots free counts own release", 1000, 5, 1, 4, 4},
{"full board degrades to one", 1000, 5, 4, 4, 1},
{"shrunk cap mid-flight clamps to one", 1000, 5, 7, 4, 1},
{"unknown slot count is conservative", 1000, 5, -1, 4, 1},
{"single-slot cap never fans out", 1000, 5, 0, 1, 1},
{"exact multiple of batch size", 15, 5, 1, 4, 3},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := followUpTriggerCount(tt.pending, tt.batchSize, tt.active, tt.max)
require.Equal(t, tt.want, got)
})
}
}
// --- activeInflightSlots ----------------------------------------------------
func TestActiveInflightSlotsPurgesExpiredAndCountsLive(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
svc := &wikiIngestService{redisClient: rdb}
ctx := context.Background()
key := wikiInflightPrefix + "kb-1"
// Two slots that expired long ago plus one live slot.
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
Score: float64(time.Now().Add(-time.Hour).UnixMilli()),
Member: "expired-1",
}).Err())
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
Score: float64(time.Now().Add(-time.Minute).UnixMilli()),
Member: "expired-2",
}).Err())
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
Score: float64(time.Now().Add(wikiInflightTTL).UnixMilli()),
Member: "live-1",
}).Err())
require.Equal(t, 1, svc.activeInflightSlots(ctx, "kb-1"))
}
func TestActiveInflightSlotsUnknownOnRedisError(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
require.NoError(t, rdb.Close()) // every Eval now fails
svc := &wikiIngestService{redisClient: rdb}
require.Equal(t, -1, svc.activeInflightSlots(context.Background(), "kb-1"))
}
// --- scheduleFollowUp -------------------------------------------------------
func newFollowUpTestService(t *testing.T, pending int64) (*wikiIngestService, *followUpEnqueuerRecorder) {
t.Helper()
rec := &followUpEnqueuerRecorder{failFrom: -1}
svc := &wikiIngestService{
task: rec,
pendingRepo: &wikiPendingRepoForFollowUpTest{pending: pending},
}
return svc, rec
}
// reserveSlots claims n live slots the way n concurrent batches would, so the
// fan-out computation sees a realistic board.
func reserveSlots(t *testing.T, svc *wikiIngestService, kbID string, n, maxInflight int) {
t.Helper()
for i := 0; i < n; i++ {
release, granted := svc.reserveInflightSlot(context.Background(), kbID, maxInflight)
require.True(t, granted)
t.Cleanup(release)
}
}
func TestScheduleFollowUpFansOutToFreeSlots(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
// 100 pending docs, batch size 10 → 10 batches needed. Two slots held
// (one of them modelling the caller's own, which releases on return),
// cap 4 → free = 4-2+1 = 3 follow-ups.
svc, rec := newFollowUpTestService(t, 100)
svc.redisClient = rdb
reserveSlots(t, svc, "kb-1", 2, 4)
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.True(t, got)
require.Equal(t, 3, rec.count())
rec.payloads(t, "kb-1")
}
func TestScheduleFollowUpBacklogSmallerThanBatchStaysSingle(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
svc, rec := newFollowUpTestService(t, 7)
svc.redisClient = rdb
reserveSlots(t, svc, "kb-1", 1, 4)
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.True(t, got)
require.Equal(t, 1, rec.count())
}
func TestScheduleFollowUpRateLimitedStaysSingle(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
// Even with the whole board free and a huge backlog, a rate-limited exit
// must not widen concurrency: widening during a 429 storm amplifies it.
svc, rec := newFollowUpTestService(t, 1000)
svc.redisClient = rdb
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiRateLimitBackoff, 10, 32, true)
require.True(t, got)
require.Equal(t, 1, rec.count())
}
func TestScheduleFollowUpLiteModeStaysSingle(t *testing.T) {
// No Redis client → liteLocks serializes batches per KB; extra triggers
// would only bounce off ErrWikiIngestConcurrent.
svc, rec := newFollowUpTestService(t, 1000)
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 32, false)
require.True(t, got)
require.Equal(t, 1, rec.count())
}
func TestScheduleFollowUpNoPendingSchedulesNothing(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
svc, rec := newFollowUpTestService(t, 0)
svc.redisClient = rdb
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.False(t, got)
require.Equal(t, 0, rec.count())
}
func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
svc, rec := newFollowUpTestService(t, 100)
svc.redisClient = rdb
rec.failFrom = 1
rec.failErr = errors.New("enqueue failed")
// The caller holds its own slot (releases on return): free = 4-1+1 = 4.
// The first enqueue succeeds, the rest fail; "scheduled" must still be
// true because at least one trigger got out.
reserveSlots(t, svc, "kb-1", 1, 4)
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.True(t, got)
require.Equal(t, 4, rec.count())
}
// --- convergence ------------------------------------------------------------
// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
// fanning out on a board whose slots never free (worst case for over-issue)
// and asserts the per-completion trigger count never exceeds the free slots,
// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
const (
maxInflight = 8
batchSize = 10
)
var (
pending = 1000
activeSlots = 1 // the completing batch itself; releases on return
)
seen := map[int]bool{}
for round := 0; round < 6; round++ {
n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
require.Greater(t, n, 0)
require.LessOrEqual(t, n, maxInflight-activeSlots+1,
"round %d: supply must not exceed free slots", round)
if activeSlots == maxInflight {
require.Equal(t, 1, n,
"round %d: a full board must clamp supply to the follow-up chain", round)
}
seen[n] = true
// The n triggers fire: n-1 of them claim slots (one bounces off the
// cap into scheduleCappedRetry when the board is full), each claims a
// batch of batchSize rows.
newBatches := min(n, maxInflight-activeSlots)
pending -= newBatches * batchSize
activeSlots = maxInflight
if pending <= 0 {
pending = 1 // keep the loop meaningful; count stays capped below
}
}
// The first round must have fanned out beyond the follow-up chain —
// otherwise this would not be testing convergence at all.
require.Greater(t, len(seen), 1)
}