fix(wiki): fan out follow-up triggers so backlogs drain at ingest_max_inflight
Since Phase 3 the consumption side (disjoint claiming, per-slug locks, the inflight cap) fully supports concurrent batches, but nothing created more than one trigger per completed batch: once uploads stopped, a KB with a standing backlog drained at 1 batch x batch-size per cycle regardless of ingest_max_inflight, leaving the wiki pool and the model server idle. followUpTriggerCount now re-arms min(batches the remaining rows still need, slots that could be free when the triggers fire). The caller's own slot releases right after scheduleFollowUp returns, hence the +1. activeInflightSlots is a read-mostly Lua companion to the reserve script (purge expired, then ZCARD); -1 means unknown (no Redis / Redis error) and collapses to a single follow-up, as do maxInflight <= 1 and Lite mode (liteLocks serialize per KB anyway). Fan-out is deliberately skipped on a rate-limited exit so a 429 storm is not amplified. Over-issued triggers are harmless: the ones that cannot get a slot are acked and coalesced by scheduleCappedRetry's fixed TaskID. The slot count renders as "unknown" in the log when it was not consulted.main
parent
bad7afa6ae
commit
4ba85e2edf
|
|
@ -1112,6 +1112,34 @@ func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string
|
|||
}, true
|
||||
}
|
||||
|
||||
// wikiInflightCountScript purges expired slots exactly like the reserve
|
||||
// script does, then returns the live count. Read-mostly companion to
|
||||
// wikiInflightReserveScript so a fan-out decision sees the same view of the
|
||||
// cap a reserver would.
|
||||
const wikiInflightCountScript = `
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, tonumber(ARGV[1]))
|
||||
return redis.call('ZCARD', KEYS[1])
|
||||
`
|
||||
|
||||
// activeInflightSlots reports how many batches currently hold one of KB's
|
||||
// in-flight slots (standard/Redis mode). Returns -1 when the count is
|
||||
// unavailable (no Redis client, or a Redis error); callers must treat that as
|
||||
// "unknown" and stay conservative instead of scaling on a stale number.
|
||||
func (s *wikiIngestService) activeInflightSlots(ctx context.Context, kbID string) int {
|
||||
if s.redisClient == nil {
|
||||
return -1
|
||||
}
|
||||
res, err := s.redisClient.Eval(ctx, wikiInflightCountScript,
|
||||
[]string{wikiInflightPrefix + kbID},
|
||||
time.Now().UnixMilli(),
|
||||
).Int()
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: inflight count failed for KB %s: %v", kbID, err)
|
||||
return -1
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// scheduleCappedRetry enqueues a single coalesced follow-up trigger after a
|
||||
// batch was turned away by the in-flight cap. asynq.TaskID collapses all
|
||||
// turned-away triggers for one KB into a single pending retry (no thundering
|
||||
|
|
|
|||
|
|
@ -23,23 +23,68 @@ import (
|
|||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// scheduleFollowUp enqueues another asynq trigger task if there are
|
||||
// still pending ops in task_pending_ops for this KB. Returns true when
|
||||
// a follow-up was scheduled.
|
||||
// followUpTriggerCount decides how many follow-up triggers a completed batch
|
||||
// should re-arm for its KB (#3461). Since Phase 3 the consumption side
|
||||
// (disjoint claiming, per-slug locks, the inflight cap) fully supports
|
||||
// concurrent batches, but nothing created more than one trigger for a KB with
|
||||
// a standing backlog — so once uploads stopped, the backlog drained at
|
||||
// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
|
||||
// the wiki pool and the model server idle.
|
||||
//
|
||||
// Post-Phase-3 this only backstops the case where a batch drained its
|
||||
// claimed window but more rows remain and no other trigger is pending
|
||||
// (e.g. steady trickle of uploads). Standard mode already fans a KB's
|
||||
// backlog across concurrent claiming batches, so the short delay is
|
||||
// normally just a light debounce rather than a lock-release wait.
|
||||
// The count is the smaller of (a) how many batches the remaining rows still
|
||||
// need and (b) how many slots could be free by the time the triggers fire:
|
||||
// the caller's own slot releases right after it returns, hence the +1.
|
||||
// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
|
||||
// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
|
||||
// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
|
||||
// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
|
||||
func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
|
||||
if pending <= 0 || batchSize <= 0 {
|
||||
return 0
|
||||
}
|
||||
batchesNeeded := (pending + batchSize - 1) / batchSize
|
||||
if activeSlots < 0 || maxInflight <= 1 {
|
||||
return 1
|
||||
}
|
||||
freeSlots := maxInflight - activeSlots + 1 // caller's slot releases on return
|
||||
if freeSlots < 1 {
|
||||
freeSlots = 1
|
||||
}
|
||||
return min(batchesNeeded, freeSlots)
|
||||
}
|
||||
|
||||
// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
|
||||
// ops in task_pending_ops for this KB. Returns true when at least one
|
||||
// follow-up was scheduled.
|
||||
//
|
||||
// `delay` is the ProcessIn before the follow-up fires. Callers pass
|
||||
// Post-Phase-3 this backstops the case where a batch drained its claimed
|
||||
// window but more rows remain and no other trigger is pending (e.g. a steady
|
||||
// trickle of uploads). Standard mode already fans a KB's backlog across
|
||||
// concurrent claiming batches, so the short delay is normally just a light
|
||||
// debounce rather than a lock-release wait.
|
||||
//
|
||||
// How many triggers are re-armed follows followUpTriggerCount above: enough
|
||||
// to drain the backlog, capped by the free inflight slots, so the concurrent
|
||||
// batch count converges on ingest_max_inflight instead of idling at 1.
|
||||
//
|
||||
// Fan-out is deliberately skipped when rateLimited: widening concurrency
|
||||
// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
|
||||
// never fans out — liteLocks serializes batches per KB, so extra triggers
|
||||
// would only bounce off ErrWikiIngestConcurrent.
|
||||
//
|
||||
// `delay` is the ProcessIn before each follow-up fires. Callers pass
|
||||
// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
|
||||
// batch tripped an upstream rate limit — the released failed rows are
|
||||
// eligible immediately, but nothing claims them until a trigger fires, so
|
||||
// stretching the follow-up interval is what actually paces retries down
|
||||
// during a 429 storm.
|
||||
func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool {
|
||||
func (s *wikiIngestService) scheduleFollowUp(
|
||||
ctx context.Context,
|
||||
payload WikiIngestPayload,
|
||||
delay time.Duration,
|
||||
batchSize, maxInflight int,
|
||||
rateLimited bool,
|
||||
) bool {
|
||||
if s.pendingRepo == nil {
|
||||
return false
|
||||
}
|
||||
|
|
@ -48,21 +93,39 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
|
|||
return false
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay)
|
||||
// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
|
||||
// a rate-limited exit; -1 also shows up in the log line as "unknown".
|
||||
activeSlots := -1
|
||||
if !rateLimited {
|
||||
activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
|
||||
}
|
||||
n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
|
||||
|
||||
activeStr := fmt.Sprintf("%d", activeSlots)
|
||||
if activeSlots < 0 {
|
||||
activeStr = "unknown"
|
||||
}
|
||||
logger.Infof(ctx,
|
||||
"wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
|
||||
count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
|
||||
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
payloadBytes, _ := json.Marshal(payload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)
|
||||
return false
|
||||
scheduled := false
|
||||
for i := 0; i < n; i++ {
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
|
||||
continue
|
||||
}
|
||||
scheduled = true
|
||||
}
|
||||
return true
|
||||
return scheduled
|
||||
}
|
||||
|
||||
// newWikiBatchContext builds the per-run lazy fetchers used by both the ingest
|
||||
|
|
@ -946,7 +1009,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
|||
logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay)
|
||||
}
|
||||
followCtx, followCancel := wikiIngestCleanupContext(ctx)
|
||||
followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay)
|
||||
followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay, batchSize, maxInflight, rateLimited)
|
||||
followCancel()
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,294 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- test doubles -----------------------------------------------------------
|
||||
|
||||
// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
|
||||
// PendingCount configurable, which is the only input scheduleFollowUp reads.
|
||||
type wikiPendingRepoForFollowUpTest struct {
|
||||
wikiPendingRepoForCleanupTest
|
||||
pending int64
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForFollowUpTest) PendingCount(
|
||||
context.Context, string, string, string,
|
||||
) (int64, error) {
|
||||
return r.pending, nil
|
||||
}
|
||||
|
||||
// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
|
||||
// failFrom >= 0 makes enqueues from that index onward return failErr so the
|
||||
// partial-failure path can be exercised.
|
||||
type followUpEnqueuerRecorder struct {
|
||||
mu sync.Mutex
|
||||
tasks []*asynq.Task
|
||||
failFrom int
|
||||
failErr error
|
||||
}
|
||||
|
||||
func (e *followUpEnqueuerRecorder) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
idx := len(e.tasks)
|
||||
e.tasks = append(e.tasks, task)
|
||||
if e.failFrom >= 0 && idx >= e.failFrom {
|
||||
return nil, e.failErr
|
||||
}
|
||||
return &asynq.TaskInfo{}, nil
|
||||
}
|
||||
|
||||
func (e *followUpEnqueuerRecorder) count() int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return len(e.tasks)
|
||||
}
|
||||
|
||||
func (e *followUpEnqueuerRecorder) payloads(t *testing.T, kbID string) {
|
||||
t.Helper()
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
for _, task := range e.tasks {
|
||||
var payload WikiIngestPayload
|
||||
require.NoError(t, json.Unmarshal(task.Payload(), &payload))
|
||||
require.Equal(t, kbID, payload.KnowledgeBaseID)
|
||||
}
|
||||
}
|
||||
|
||||
// --- pure fan-out computation ----------------------------------------------
|
||||
|
||||
func TestFollowUpTriggerCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pending, batchSize, active, max int
|
||||
want int
|
||||
}{
|
||||
{"no pending enqueues nothing", 0, 5, 1, 4, 0},
|
||||
{"defensive zero batch size", 10, 0, 1, 4, 0},
|
||||
{"one batch or less stays at one", 5, 5, 1, 4, 1},
|
||||
{"single row stays at one", 1, 5, 1, 4, 1},
|
||||
{"just over one batch needs two", 6, 5, 1, 4, 2},
|
||||
{"backlog need is capped by free slots", 1000, 5, 2, 4, 3},
|
||||
{"all slots free counts own release", 1000, 5, 1, 4, 4},
|
||||
{"full board degrades to one", 1000, 5, 4, 4, 1},
|
||||
{"shrunk cap mid-flight clamps to one", 1000, 5, 7, 4, 1},
|
||||
{"unknown slot count is conservative", 1000, 5, -1, 4, 1},
|
||||
{"single-slot cap never fans out", 1000, 5, 0, 1, 1},
|
||||
{"exact multiple of batch size", 15, 5, 1, 4, 3},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := followUpTriggerCount(tt.pending, tt.batchSize, tt.active, tt.max)
|
||||
require.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- activeInflightSlots ----------------------------------------------------
|
||||
|
||||
func TestActiveInflightSlotsPurgesExpiredAndCountsLive(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
svc := &wikiIngestService{redisClient: rdb}
|
||||
ctx := context.Background()
|
||||
key := wikiInflightPrefix + "kb-1"
|
||||
|
||||
// Two slots that expired long ago plus one live slot.
|
||||
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
|
||||
Score: float64(time.Now().Add(-time.Hour).UnixMilli()),
|
||||
Member: "expired-1",
|
||||
}).Err())
|
||||
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
|
||||
Score: float64(time.Now().Add(-time.Minute).UnixMilli()),
|
||||
Member: "expired-2",
|
||||
}).Err())
|
||||
require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
|
||||
Score: float64(time.Now().Add(wikiInflightTTL).UnixMilli()),
|
||||
Member: "live-1",
|
||||
}).Err())
|
||||
|
||||
require.Equal(t, 1, svc.activeInflightSlots(ctx, "kb-1"))
|
||||
}
|
||||
|
||||
func TestActiveInflightSlotsUnknownOnRedisError(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
require.NoError(t, rdb.Close()) // every Eval now fails
|
||||
|
||||
svc := &wikiIngestService{redisClient: rdb}
|
||||
require.Equal(t, -1, svc.activeInflightSlots(context.Background(), "kb-1"))
|
||||
}
|
||||
|
||||
// --- scheduleFollowUp -------------------------------------------------------
|
||||
|
||||
func newFollowUpTestService(t *testing.T, pending int64) (*wikiIngestService, *followUpEnqueuerRecorder) {
|
||||
t.Helper()
|
||||
rec := &followUpEnqueuerRecorder{failFrom: -1}
|
||||
svc := &wikiIngestService{
|
||||
task: rec,
|
||||
pendingRepo: &wikiPendingRepoForFollowUpTest{pending: pending},
|
||||
}
|
||||
return svc, rec
|
||||
}
|
||||
|
||||
// reserveSlots claims n live slots the way n concurrent batches would, so the
|
||||
// fan-out computation sees a realistic board.
|
||||
func reserveSlots(t *testing.T, svc *wikiIngestService, kbID string, n, maxInflight int) {
|
||||
t.Helper()
|
||||
for i := 0; i < n; i++ {
|
||||
release, granted := svc.reserveInflightSlot(context.Background(), kbID, maxInflight)
|
||||
require.True(t, granted)
|
||||
t.Cleanup(release)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpFansOutToFreeSlots(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
// 100 pending docs, batch size 10 → 10 batches needed. Two slots held
|
||||
// (one of them modelling the caller's own, which releases on return),
|
||||
// cap 4 → free = 4-2+1 = 3 follow-ups.
|
||||
svc, rec := newFollowUpTestService(t, 100)
|
||||
svc.redisClient = rdb
|
||||
reserveSlots(t, svc, "kb-1", 2, 4)
|
||||
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 3, rec.count())
|
||||
rec.payloads(t, "kb-1")
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpBacklogSmallerThanBatchStaysSingle(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
svc, rec := newFollowUpTestService(t, 7)
|
||||
svc.redisClient = rdb
|
||||
reserveSlots(t, svc, "kb-1", 1, 4)
|
||||
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 1, rec.count())
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpRateLimitedStaysSingle(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
// Even with the whole board free and a huge backlog, a rate-limited exit
|
||||
// must not widen concurrency: widening during a 429 storm amplifies it.
|
||||
svc, rec := newFollowUpTestService(t, 1000)
|
||||
svc.redisClient = rdb
|
||||
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiRateLimitBackoff, 10, 32, true)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 1, rec.count())
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpLiteModeStaysSingle(t *testing.T) {
|
||||
// No Redis client → liteLocks serializes batches per KB; extra triggers
|
||||
// would only bounce off ErrWikiIngestConcurrent.
|
||||
svc, rec := newFollowUpTestService(t, 1000)
|
||||
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 32, false)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 1, rec.count())
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpNoPendingSchedulesNothing(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
svc, rec := newFollowUpTestService(t, 0)
|
||||
svc.redisClient = rdb
|
||||
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
|
||||
require.False(t, got)
|
||||
require.Equal(t, 0, rec.count())
|
||||
}
|
||||
|
||||
func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
|
||||
svc, rec := newFollowUpTestService(t, 100)
|
||||
svc.redisClient = rdb
|
||||
rec.failFrom = 1
|
||||
rec.failErr = errors.New("enqueue failed")
|
||||
|
||||
// The caller holds its own slot (releases on return): free = 4-1+1 = 4.
|
||||
// The first enqueue succeeds, the rest fail; "scheduled" must still be
|
||||
// true because at least one trigger got out.
|
||||
reserveSlots(t, svc, "kb-1", 1, 4)
|
||||
got := svc.scheduleFollowUp(context.Background(),
|
||||
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
|
||||
require.True(t, got)
|
||||
require.Equal(t, 4, rec.count())
|
||||
}
|
||||
|
||||
// --- convergence ------------------------------------------------------------
|
||||
|
||||
// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
|
||||
// fanning out on a board whose slots never free (worst case for over-issue)
|
||||
// and asserts the per-completion trigger count never exceeds the free slots,
|
||||
// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
|
||||
func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
|
||||
const (
|
||||
maxInflight = 8
|
||||
batchSize = 10
|
||||
)
|
||||
var (
|
||||
pending = 1000
|
||||
activeSlots = 1 // the completing batch itself; releases on return
|
||||
)
|
||||
seen := map[int]bool{}
|
||||
for round := 0; round < 6; round++ {
|
||||
n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
|
||||
require.Greater(t, n, 0)
|
||||
require.LessOrEqual(t, n, maxInflight-activeSlots+1,
|
||||
"round %d: supply must not exceed free slots", round)
|
||||
if activeSlots == maxInflight {
|
||||
require.Equal(t, 1, n,
|
||||
"round %d: a full board must clamp supply to the follow-up chain", round)
|
||||
}
|
||||
seen[n] = true
|
||||
|
||||
// The n triggers fire: n-1 of them claim slots (one bounces off the
|
||||
// cap into scheduleCappedRetry when the board is full), each claims a
|
||||
// batch of batchSize rows.
|
||||
newBatches := min(n, maxInflight-activeSlots)
|
||||
pending -= newBatches * batchSize
|
||||
activeSlots = maxInflight
|
||||
if pending <= 0 {
|
||||
pending = 1 // keep the loop meaningful; count stays capped below
|
||||
}
|
||||
}
|
||||
// The first round must have fanned out beyond the follow-up chain —
|
||||
// otherwise this would not be testing convergence at all.
|
||||
require.Greater(t, len(seen), 1)
|
||||
}
|
||||
Loading…
Reference in New Issue