From c57766cc76061acb4fdd2668ca0eb5e76aa4d015 Mon Sep 17 00:00:00 2001 From: LuckVd <37114923+LuckVd@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:13:32 +0800 Subject: [PATCH] fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests (#3602) * fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests Tenant deletion only soft-deletes the tenant and its memberships; its knowledge bases, model configs, and durable task_pending_ops rows survive. Wiki ingest/finalize tasks restored from those ops only checked KB state, so a deleted tenant kept accruing new LLM requests (observed as ~11h of continued synthesis calls in #3593). Gate every task entry point on tenant liveness: - EnqueueIfKnowledgeBaseActive now also requires tenants.deleted_at IS NULL inside the same transaction (fail closed). - Startup recovery purges pending rows whose tenant is soft-deleted (the predicate is wrapped in parentheses: NOT EXISTS(...) AND EXISTS(...) would parse as (NOT EXISTS(...)) AND EXISTS(...) and silently match nothing). - Wiki ingest and finalize handlers check tenant liveness from the payload before any model call and drop the KB's queue when the tenant is gone. Liveness lookup errors fail open; the guarded enqueue and recovery paths enforce the same invariant on their own DB access. Adds TaskPendingOpsTenantLiveness as an optional extension interface, regression tests for the enqueue guard, startup recovery, and the task entry guards. Fixes #3593 * style(container): wrap long log line over 120 chars for golangci-lint (lll) --------- Co-authored-by: LuckVd --- internal/application/repository/task_queue.go | 41 +++++++- .../application/repository/task_queue_test.go | 25 ++++- .../service/wiki_deleted_tenant_guard_test.go | 93 +++++++++++++++++++ internal/application/service/wiki_ingest.go | 25 +++++ .../application/service/wiki_ingest_batch.go | 22 +++++ .../container/recover_pending_wiki_tasks.go | 20 +++- .../container/reset_pending_tasks_test.go | 29 +++++- internal/types/interfaces/task_queue.go | 10 ++ 8 files changed, 250 insertions(+), 15 deletions(-) create mode 100644 internal/application/service/wiki_deleted_tenant_guard_test.go diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index 1823732c..f35b7212 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -53,11 +53,13 @@ func preparePendingOp(op *types.TaskPendingOp) error { } // EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new -// durable work after a KB was soft-deleted. On Postgres the share lock -// serializes this check+insert transaction against the row update performed by -// soft deletion: whichever operation acquires the row first determines the -// order, and the deletion path's subsequent scope scrub removes any insert -// that committed before it. +// durable work after a KB was soft-deleted. The tenant must also be alive: a +// tenant soft-deletion removes the workspace without touching its knowledge +// bases, and a deleted tenant must never accrue new model-backed work. On +// Postgres the share lock serializes this check+insert transaction against +// the row update performed by soft deletion: whichever operation acquires the +// row first determines the order, and the deletion path's subsequent scope +// scrub removes any insert that committed before it. func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( ctx context.Context, op *types.TaskPendingOp, @@ -84,6 +86,10 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( } return err } + active, err := tenantActiveWithinTx(tx, op.TenantID) + if err != nil || !active { + return err + } if err := tx.Create(op).Error; err != nil { return err } @@ -93,6 +99,31 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( return accepted, err } +// tenantActiveWithinTx reports whether the tenant row exists and is not +// soft-deleted, inside the caller's transaction. Callers that cannot see a +// tenants table (legacy test doubles) fail closed. +func tenantActiveWithinTx(tx *gorm.DB, tenantID uint64) (bool, error) { + if tenantID == 0 { + return false, nil + } + var tenant types.Tenant + err := tx.Model(&types.Tenant{}).Select("id").Where("id = ?", tenantID).Take(&tenant).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// HasActiveTenant reports whether the tenant exists and has not been +// soft-deleted. Wiki task consumers call this before doing durable or +// model-backed work on the tenant's behalf (#3593). +func (r *taskPendingOpsRepository) HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) { + return tenantActiveWithinTx(r.db.WithContext(ctx), tenantID) +} + // SeedKnowledgeFinalizingWithPendingOp commits the finalizing counter and the // durable operation that owns one slot in the same transaction. This closes // the crash window where a knowledge row could enter finalizing before its diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index 87d9bf18..dc8fe547 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -392,15 +392,32 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) { deleted_at DATETIME )`).Error) require.NoError(t, db.Exec( - "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)", - "kb-active", 1, "kb-deleted", 1, time.Now(), + "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?), (?, ?, NULL)", + "kb-active", 1, "kb-deleted", 1, time.Now(), "kb-t2", 2, + ).Error) + require.NoError(t, db.Exec(`CREATE TABLE tenants ( + id INTEGER PRIMARY KEY, + deleted_at DATETIME + )`).Error) + require.NoError(t, db.Exec( + "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, ?)", + 1, 2, time.Now(), ).Error) repo := NewTaskPendingOpsRepository(db) guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard) require.True(t, ok) + liveness, ok := repo.(interfaces.TaskPendingOpsTenantLiveness) + require.True(t, ok, "task pending repository must expose tenant liveness for wiki task guards") ctx := context.Background() + activeTenant, err := liveness.HasActiveTenant(ctx, 1) + require.NoError(t, err) + assert.True(t, activeTenant) + deletedTenant, err := liveness.HasActiveTenant(ctx, 2) + require.NoError(t, err) + assert.False(t, deletedTenant) + accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil)) require.NoError(t, err) @@ -426,6 +443,10 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) { TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase, ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant", }}, + {name: "deleted tenant with live KB", op: &types.TaskPendingOp{ + TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase, + ScopeID: "kb-t2", Op: "ingest", DedupKey: "deleted-tenant", + }}, } { t.Run(tc.name, func(t *testing.T) { accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op) diff --git a/internal/application/service/wiki_deleted_tenant_guard_test.go b/internal/application/service/wiki_deleted_tenant_guard_test.go new file mode 100644 index 00000000..d84c6944 --- /dev/null +++ b/internal/application/service/wiki_deleted_tenant_guard_test.go @@ -0,0 +1,93 @@ +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wikiTenantGuardPendingRepo stubs the slice of TaskPendingOpsRepository the +// tenant-liveness guard touches: the liveness lookup and the scope cleanup. +type wikiTenantGuardPendingRepo struct { + interfaces.TaskPendingOpsRepository + active bool + lookupErr error + lookups []uint64 + deletedKBs []string +} + +func (r *wikiTenantGuardPendingRepo) HasActiveTenant(_ context.Context, tenantID uint64) (bool, error) { + r.lookups = append(r.lookups, tenantID) + return r.active, r.lookupErr +} + +func (r *wikiTenantGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error { + if scope == types.TaskScopeKnowledgeBase { + r.deletedKBs = append(r.deletedKBs, scopeID) + } + return nil +} + +func TestWikiTaskSkipsDeletedTenant(t *testing.T) { + payload, err := json.Marshal(WikiIngestPayload{ + TenantID: 7, + KnowledgeBaseID: "kb-orphaned", + Language: "en", + }) + require.NoError(t, err) + + t.Run("ingest clears queue and does not proceed", func(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload)) + require.NoError(t, err) + require.Len(t, repo.lookups, 1) + assert.Equal(t, uint64(7), repo.lookups[0]) + assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs) + }) + + t.Run("finalize clears queue and does not proceed", func(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.ProcessWikiFinalize(context.Background(), asynq.NewTask(types.TypeWikiFinalize, payload)) + require.NoError(t, err) + require.Len(t, repo.lookups, 1) + assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs) + }) +} + +func TestWikiTenantGuardFailsOpenOnLookupError(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false, lookupErr: assert.AnError} + svc := &wikiIngestService{pendingRepo: repo} + + // A transient liveness-lookup failure must not be read as "deleted": + // the task keeps its retry budget and the enqueue guard / startup + // recovery enforce the same invariant on their own DB access. + assert.False(t, svc.tenantIsDeleted(context.Background(), 7)) + assert.Empty(t, repo.deletedKBs) +} + +func TestWikiTenantGuardWithoutLivenessCapability(t *testing.T) { + // A pending repo without the liveness extension (legacy test doubles, + // alternate implementations) is never treated as a deleted tenant. + svc := &wikiIngestService{pendingRepo: &wikiKBGuardPendingRepo{}} + + assert.False(t, svc.tenantIsDeleted(context.Background(), 7)) +} + +func TestWikiTenantGuardIgnoresZeroTenant(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + assert.False(t, svc.tenantIsDeleted(context.Background(), 0)) + assert.Empty(t, repo.lookups) + _ = interfaces.TaskPendingOpsTenantLiveness(nil) +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index da081a5a..097d85f8 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -720,6 +720,31 @@ func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Cont return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID) } +// tenantIsDeleted reports whether the payload's tenant has been soft-deleted. +// A tenant deletion removes the workspace but leaves its knowledge bases and +// durable pending ops in place, so wiki tasks restored from those ops would +// otherwise keep issuing model requests for a tenant nobody can reach (#3593). +// +// Fail-open on transient lookup errors and when the pending repo does not +// expose tenant liveness (legacy test doubles): the retry machinery still +// covers the task, and the guarded enqueue / startup recovery paths enforce +// the same invariant on their own DB access. +func (s *wikiIngestService) tenantIsDeleted(ctx context.Context, tenantID uint64) bool { + if tenantID == 0 { + return false + } + checker, ok := s.pendingRepo.(interfaces.TaskPendingOpsTenantLiveness) + if !ok { + return false + } + active, err := checker.HasActiveTenant(ctx, tenantID) + if err != nil { + logger.Warnf(ctx, "wiki: tenant liveness lookup failed for tenant %d: %v (failing open)", tenantID, err) + return false + } + return !active +} + // releaseIngestForUnavailableWiki drops the KB's queued ingest ops when the // wiki cannot run for a reason retries will not fix, and releases each // document's wiki slot in the same transaction so it leaves "finalizing". diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index d2d79414..f1d92ad5 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -254,6 +254,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } + // A soft-deleted tenant owns no reachable workspace anymore; its KBs and + // durable pending ops survive the deletion, so without this guard the + // batch would keep issuing model requests (and finalize would rebuild + // index pages) for a tenant nobody can see (#3593). Drop the KB's queue. + if s.tenantIsDeleted(ctx, payload.TenantID) { + exitStatus = "tenant_deleted" + if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil { + return fmt.Errorf("wiki ingest: clear deleted tenant queue: %w", err) + } + return nil + } + // Concurrency model (Phase 3): // // - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches @@ -931,6 +943,16 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta return nil } + // Same tenant-liveness guard as the ingest batch: finalize calls the + // synthesis model to rebuild the index page, and a deleted tenant must + // never accrue new model requests (#3593). Drop the KB's queue. + if s.tenantIsDeleted(ctx, payload.TenantID) { + if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil { + return fmt.Errorf("wiki finalize: clear deleted tenant queue: %w", err) + } + return nil + } + // Per-KB finalize lock, separate from the ingest active lock so finalize // and ingest batches never block each other. Coalescing via asynq.TaskID // already keeps at most one finalize pending per KB; this guards the diff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.go index 1867e055..adf24202 100644 --- a/internal/container/recover_pending_wiki_tasks.go +++ b/internal/container/recover_pending_wiki_tasks.go @@ -34,16 +34,24 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { return } ctx := context.Background() - const activeKnowledgeBase = `EXISTS ( + // The outer parentheses matter: the cleanup below negates this whole + // predicate with NOT, and `NOT EXISTS(...) AND EXISTS(...)` would parse + // as `(NOT EXISTS(...)) AND EXISTS(...)` — silently matching nothing. + const activeKnowledgeBase = `(EXISTS ( SELECT 1 FROM knowledge_bases kb WHERE kb.id = task_pending_ops.scope_id AND kb.tenant_id = task_pending_ops.tenant_id AND kb.deleted_at IS NULL - )` + ) AND EXISTS ( + SELECT 1 FROM tenants t + WHERE t.id = task_pending_ops.tenant_id + AND t.deleted_at IS NULL + ))` wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize} - // Durable rows for a deleted/missing KB must not recreate ephemeral - // triggers at startup. Fail closed if this cleanup cannot be verified. + // Durable rows for a deleted/missing KB — or a soft-deleted tenant — + // must not recreate ephemeral triggers at startup. Fail closed if this + // cleanup cannot be verified. cleanup := db.WithContext(ctx). Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes). Where("NOT " + activeKnowledgeBase). @@ -53,7 +61,9 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { return } if cleanup.RowsAffected > 0 { - logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected) + logger.Infof(ctx, + "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases or tenants", + cleanup.RowsAffected) } var scopes []pendingWikiScope diff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.go index 47d4c024..4df02aa0 100644 --- a/internal/container/reset_pending_tasks_test.go +++ b/internal/container/reset_pending_tasks_test.go @@ -89,6 +89,13 @@ CREATE TABLE IF NOT EXISTS knowledge_bases ( ); ` +var resetPendingTenantsDDL = ` +CREATE TABLE IF NOT EXISTS tenants ( + id INTEGER PRIMARY KEY, + deleted_at DATETIME +); +` + func setupResetPendingDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) @@ -98,6 +105,14 @@ func setupResetPendingDB(t *testing.T) *gorm.DB { require.NoError(t, db.Exec(resetPendingSpansDDL).Error) require.NoError(t, db.Exec(resetPendingOpsDDL).Error) require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error) + require.NoError(t, db.Exec(resetPendingTenantsDDL).Error) + // Tenants 7 and 8 are alive; tenant 9 is soft-deleted — its KB-scoped + // pending rows must be purged by recovery even though the KB row is + // still present (#3593). + require.NoError(t, db.Exec( + "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, NULL), (?, ?)", + 7, 8, 9, time.Now(), + ).Error) return db } @@ -309,8 +324,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { db := setupResetPendingDB(t) require.NoError(t, db.Exec( `INSERT INTO knowledge_bases (id, tenant_id, deleted_at) - VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`, - "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), + VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?), (?, ?, NULL)`, + "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), "kb-t9", 9, ).Error) rows := []struct { tenantID uint64 @@ -323,6 +338,7 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { {7, types.TypeWikiFinalize, "kb-a", "slug-a"}, {8, types.TypeWikiIngest, "kb-b", "k-3"}, {9, types.TypeWikiIngest, "kb-deleted", "k-deleted"}, + {9, types.TypeWikiIngest, "kb-t9", "k-t9"}, // live KB of a deleted tenant {10, types.TypeWikiFinalize, "kb-missing", "k-missing"}, } for _, row := range rows { @@ -350,7 +366,14 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { var orphaned int64 require.NoError(t, db.Model(&types.TaskPendingOp{}). - Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}). + Where("scope_id IN ?", []string{"kb-deleted", "kb-missing", "kb-t9"}). Count(&orphaned).Error) assert.Zero(t, orphaned) + + // The live KB of a soft-deleted tenant must be purged too: the KB row + // survives the tenant deletion, so the KB-only predicate would keep + // re-arming its triggers at every startup (#3593). + var remaining int64 + require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&remaining).Error) + assert.Equal(t, int64(4), remaining) } diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index c2fae3f0..e6d8dfd7 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -114,6 +114,16 @@ type TaskPendingOpsKnowledgeBaseGuard interface { EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error) } +// TaskPendingOpsTenantLiveness reports whether a tenant is still alive (not +// soft-deleted). Wiki task consumers use it to guarantee a deleted tenant +// never triggers new model requests: the check runs at task entry (ingest +// and finalize) and inside the guarded enqueue, alongside the KB-active +// check — tenant soft-deletion removes the workspace without touching its +// knowledge bases or durable pending ops. +type TaskPendingOpsTenantLiveness interface { + HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) +} + // TaskPendingOpsFinalizingSeeder atomically hands a processing knowledge row // to the asynchronous finalizing pipeline while persisting the durable // pending operation that owns one of its subtask slots.