fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests (#3602)

* fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests

Tenant deletion only soft-deletes the tenant and its memberships; its
knowledge bases, model configs, and durable task_pending_ops rows survive.
Wiki ingest/finalize tasks restored from those ops only checked KB state,
so a deleted tenant kept accruing new LLM requests (observed as ~11h of
continued synthesis calls in #3593).

Gate every task entry point on tenant liveness:

- EnqueueIfKnowledgeBaseActive now also requires tenants.deleted_at IS NULL
  inside the same transaction (fail closed).
- Startup recovery purges pending rows whose tenant is soft-deleted (the
  predicate is wrapped in parentheses: NOT EXISTS(...) AND EXISTS(...) would
  parse as (NOT EXISTS(...)) AND EXISTS(...) and silently match nothing).
- Wiki ingest and finalize handlers check tenant liveness from the payload
  before any model call and drop the KB's queue when the tenant is gone.
  Liveness lookup errors fail open; the guarded enqueue and recovery paths
  enforce the same invariant on their own DB access.

Adds TaskPendingOpsTenantLiveness as an optional extension interface,
regression tests for the enqueue guard, startup recovery, and the task
entry guards.

Fixes #3593

* style(container): wrap long log line over 120 chars for golangci-lint (lll)

---------

Co-authored-by: LuckVd <LuckVd@users.noreply.github.com>
main
LuckVd 2026-09-23 12:13:32 +08:00 committed by GitHub
parent 23832fabad
commit c57766cc76
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 250 additions and 15 deletions

View File

@ -53,11 +53,13 @@ func preparePendingOp(op *types.TaskPendingOp) error {
}
// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new
// durable work after a KB was soft-deleted. On Postgres the share lock
// serializes this check+insert transaction against the row update performed by
// soft deletion: whichever operation acquires the row first determines the
// order, and the deletion path's subsequent scope scrub removes any insert
// that committed before it.
// durable work after a KB was soft-deleted. The tenant must also be alive: a
// tenant soft-deletion removes the workspace without touching its knowledge
// bases, and a deleted tenant must never accrue new model-backed work. On
// Postgres the share lock serializes this check+insert transaction against
// the row update performed by soft deletion: whichever operation acquires the
// row first determines the order, and the deletion path's subsequent scope
// scrub removes any insert that committed before it.
func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
ctx context.Context,
op *types.TaskPendingOp,
@ -84,6 +86,10 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
}
return err
}
active, err := tenantActiveWithinTx(tx, op.TenantID)
if err != nil || !active {
return err
}
if err := tx.Create(op).Error; err != nil {
return err
}
@ -93,6 +99,31 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
return accepted, err
}
// tenantActiveWithinTx reports whether the tenant row exists and is not
// soft-deleted, inside the caller's transaction. Callers that cannot see a
// tenants table (legacy test doubles) fail closed.
func tenantActiveWithinTx(tx *gorm.DB, tenantID uint64) (bool, error) {
if tenantID == 0 {
return false, nil
}
var tenant types.Tenant
err := tx.Model(&types.Tenant{}).Select("id").Where("id = ?", tenantID).Take(&tenant).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
// HasActiveTenant reports whether the tenant exists and has not been
// soft-deleted. Wiki task consumers call this before doing durable or
// model-backed work on the tenant's behalf (#3593).
func (r *taskPendingOpsRepository) HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) {
return tenantActiveWithinTx(r.db.WithContext(ctx), tenantID)
}
// SeedKnowledgeFinalizingWithPendingOp commits the finalizing counter and the
// durable operation that owns one slot in the same transaction. This closes
// the crash window where a knowledge row could enter finalizing before its

View File

@ -392,15 +392,32 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
deleted_at DATETIME
)`).Error)
require.NoError(t, db.Exec(
"INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",
"kb-active", 1, "kb-deleted", 1, time.Now(),
"INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?), (?, ?, NULL)",
"kb-active", 1, "kb-deleted", 1, time.Now(), "kb-t2", 2,
).Error)
require.NoError(t, db.Exec(`CREATE TABLE tenants (
id INTEGER PRIMARY KEY,
deleted_at DATETIME
)`).Error)
require.NoError(t, db.Exec(
"INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, ?)",
1, 2, time.Now(),
).Error)
repo := NewTaskPendingOpsRepository(db)
guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)
require.True(t, ok)
liveness, ok := repo.(interfaces.TaskPendingOpsTenantLiveness)
require.True(t, ok, "task pending repository must expose tenant liveness for wiki task guards")
ctx := context.Background()
activeTenant, err := liveness.HasActiveTenant(ctx, 1)
require.NoError(t, err)
assert.True(t, activeTenant)
deletedTenant, err := liveness.HasActiveTenant(ctx, 2)
require.NoError(t, err)
assert.False(t, deletedTenant)
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))
require.NoError(t, err)
@ -426,6 +443,10 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",
}},
{name: "deleted tenant with live KB", op: &types.TaskPendingOp{
TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
ScopeID: "kb-t2", Op: "ingest", DedupKey: "deleted-tenant",
}},
} {
t.Run(tc.name, func(t *testing.T) {
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)

View File

@ -0,0 +1,93 @@
package service
import (
"context"
"encoding/json"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// wikiTenantGuardPendingRepo stubs the slice of TaskPendingOpsRepository the
// tenant-liveness guard touches: the liveness lookup and the scope cleanup.
type wikiTenantGuardPendingRepo struct {
interfaces.TaskPendingOpsRepository
active bool
lookupErr error
lookups []uint64
deletedKBs []string
}
func (r *wikiTenantGuardPendingRepo) HasActiveTenant(_ context.Context, tenantID uint64) (bool, error) {
r.lookups = append(r.lookups, tenantID)
return r.active, r.lookupErr
}
func (r *wikiTenantGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
if scope == types.TaskScopeKnowledgeBase {
r.deletedKBs = append(r.deletedKBs, scopeID)
}
return nil
}
func TestWikiTaskSkipsDeletedTenant(t *testing.T) {
payload, err := json.Marshal(WikiIngestPayload{
TenantID: 7,
KnowledgeBaseID: "kb-orphaned",
Language: "en",
})
require.NoError(t, err)
t.Run("ingest clears queue and does not proceed", func(t *testing.T) {
repo := &wikiTenantGuardPendingRepo{active: false}
svc := &wikiIngestService{pendingRepo: repo}
err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))
require.NoError(t, err)
require.Len(t, repo.lookups, 1)
assert.Equal(t, uint64(7), repo.lookups[0])
assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)
})
t.Run("finalize clears queue and does not proceed", func(t *testing.T) {
repo := &wikiTenantGuardPendingRepo{active: false}
svc := &wikiIngestService{pendingRepo: repo}
err := svc.ProcessWikiFinalize(context.Background(), asynq.NewTask(types.TypeWikiFinalize, payload))
require.NoError(t, err)
require.Len(t, repo.lookups, 1)
assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)
})
}
func TestWikiTenantGuardFailsOpenOnLookupError(t *testing.T) {
repo := &wikiTenantGuardPendingRepo{active: false, lookupErr: assert.AnError}
svc := &wikiIngestService{pendingRepo: repo}
// A transient liveness-lookup failure must not be read as "deleted":
// the task keeps its retry budget and the enqueue guard / startup
// recovery enforce the same invariant on their own DB access.
assert.False(t, svc.tenantIsDeleted(context.Background(), 7))
assert.Empty(t, repo.deletedKBs)
}
func TestWikiTenantGuardWithoutLivenessCapability(t *testing.T) {
// A pending repo without the liveness extension (legacy test doubles,
// alternate implementations) is never treated as a deleted tenant.
svc := &wikiIngestService{pendingRepo: &wikiKBGuardPendingRepo{}}
assert.False(t, svc.tenantIsDeleted(context.Background(), 7))
}
func TestWikiTenantGuardIgnoresZeroTenant(t *testing.T) {
repo := &wikiTenantGuardPendingRepo{active: false}
svc := &wikiIngestService{pendingRepo: repo}
assert.False(t, svc.tenantIsDeleted(context.Background(), 0))
assert.Empty(t, repo.lookups)
_ = interfaces.TaskPendingOpsTenantLiveness(nil)
}

View File

@ -720,6 +720,31 @@ func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Cont
return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID)
}
// tenantIsDeleted reports whether the payload's tenant has been soft-deleted.
// A tenant deletion removes the workspace but leaves its knowledge bases and
// durable pending ops in place, so wiki tasks restored from those ops would
// otherwise keep issuing model requests for a tenant nobody can reach (#3593).
//
// Fail-open on transient lookup errors and when the pending repo does not
// expose tenant liveness (legacy test doubles): the retry machinery still
// covers the task, and the guarded enqueue / startup recovery paths enforce
// the same invariant on their own DB access.
func (s *wikiIngestService) tenantIsDeleted(ctx context.Context, tenantID uint64) bool {
if tenantID == 0 {
return false
}
checker, ok := s.pendingRepo.(interfaces.TaskPendingOpsTenantLiveness)
if !ok {
return false
}
active, err := checker.HasActiveTenant(ctx, tenantID)
if err != nil {
logger.Warnf(ctx, "wiki: tenant liveness lookup failed for tenant %d: %v (failing open)", tenantID, err)
return false
}
return !active
}
// releaseIngestForUnavailableWiki drops the KB's queued ingest ops when the
// wiki cannot run for a reason retries will not fix, and releases each
// document's wiki slot in the same transaction so it leaves "finalizing".

View File

@ -254,6 +254,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// A soft-deleted tenant owns no reachable workspace anymore; its KBs and
// durable pending ops survive the deletion, so without this guard the
// batch would keep issuing model requests (and finalize would rebuild
// index pages) for a tenant nobody can see (#3593). Drop the KB's queue.
if s.tenantIsDeleted(ctx, payload.TenantID) {
exitStatus = "tenant_deleted"
if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {
return fmt.Errorf("wiki ingest: clear deleted tenant queue: %w", err)
}
return nil
}
// Concurrency model (Phase 3):
//
// - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches
@ -931,6 +943,16 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
return nil
}
// Same tenant-liveness guard as the ingest batch: finalize calls the
// synthesis model to rebuild the index page, and a deleted tenant must
// never accrue new model requests (#3593). Drop the KB's queue.
if s.tenantIsDeleted(ctx, payload.TenantID) {
if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {
return fmt.Errorf("wiki finalize: clear deleted tenant queue: %w", err)
}
return nil
}
// Per-KB finalize lock, separate from the ingest active lock so finalize
// and ingest batches never block each other. Coalescing via asynq.TaskID
// already keeps at most one finalize pending per KB; this guards the

View File

@ -34,16 +34,24 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
return
}
ctx := context.Background()
const activeKnowledgeBase = `EXISTS (
// The outer parentheses matter: the cleanup below negates this whole
// predicate with NOT, and `NOT EXISTS(...) AND EXISTS(...)` would parse
// as `(NOT EXISTS(...)) AND EXISTS(...)` — silently matching nothing.
const activeKnowledgeBase = `(EXISTS (
SELECT 1 FROM knowledge_bases kb
WHERE kb.id = task_pending_ops.scope_id
AND kb.tenant_id = task_pending_ops.tenant_id
AND kb.deleted_at IS NULL
)`
) AND EXISTS (
SELECT 1 FROM tenants t
WHERE t.id = task_pending_ops.tenant_id
AND t.deleted_at IS NULL
))`
wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize}
// Durable rows for a deleted/missing KB must not recreate ephemeral
// triggers at startup. Fail closed if this cleanup cannot be verified.
// Durable rows for a deleted/missing KB — or a soft-deleted tenant —
// must not recreate ephemeral triggers at startup. Fail closed if this
// cleanup cannot be verified.
cleanup := db.WithContext(ctx).
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
Where("NOT " + activeKnowledgeBase).
@ -53,7 +61,9 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
return
}
if cleanup.RowsAffected > 0 {
logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected)
logger.Infof(ctx,
"[WikiRecovery] removed %d pending row(s) for deleted knowledge bases or tenants",
cleanup.RowsAffected)
}
var scopes []pendingWikiScope

View File

@ -89,6 +89,13 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
);
`
var resetPendingTenantsDDL = `
CREATE TABLE IF NOT EXISTS tenants (
id INTEGER PRIMARY KEY,
deleted_at DATETIME
);
`
func setupResetPendingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@ -98,6 +105,14 @@ func setupResetPendingDB(t *testing.T) *gorm.DB {
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error)
require.NoError(t, db.Exec(resetPendingTenantsDDL).Error)
// Tenants 7 and 8 are alive; tenant 9 is soft-deleted — its KB-scoped
// pending rows must be purged by recovery even though the KB row is
// still present (#3593).
require.NoError(t, db.Exec(
"INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, NULL), (?, ?)",
7, 8, 9, time.Now(),
).Error)
return db
}
@ -309,8 +324,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
db := setupResetPendingDB(t)
require.NoError(t, db.Exec(
`INSERT INTO knowledge_bases (id, tenant_id, deleted_at)
VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`,
"kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(),
VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?), (?, ?, NULL)`,
"kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), "kb-t9", 9,
).Error)
rows := []struct {
tenantID uint64
@ -323,6 +338,7 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
{8, types.TypeWikiIngest, "kb-b", "k-3"},
{9, types.TypeWikiIngest, "kb-deleted", "k-deleted"},
{9, types.TypeWikiIngest, "kb-t9", "k-t9"}, // live KB of a deleted tenant
{10, types.TypeWikiFinalize, "kb-missing", "k-missing"},
}
for _, row := range rows {
@ -350,7 +366,14 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
var orphaned int64
require.NoError(t, db.Model(&types.TaskPendingOp{}).
Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}).
Where("scope_id IN ?", []string{"kb-deleted", "kb-missing", "kb-t9"}).
Count(&orphaned).Error)
assert.Zero(t, orphaned)
// The live KB of a soft-deleted tenant must be purged too: the KB row
// survives the tenant deletion, so the KB-only predicate would keep
// re-arming its triggers at every startup (#3593).
var remaining int64
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&remaining).Error)
assert.Equal(t, int64(4), remaining)
}

View File

@ -114,6 +114,16 @@ type TaskPendingOpsKnowledgeBaseGuard interface {
EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error)
}
// TaskPendingOpsTenantLiveness reports whether a tenant is still alive (not
// soft-deleted). Wiki task consumers use it to guarantee a deleted tenant
// never triggers new model requests: the check runs at task entry (ingest
// and finalize) and inside the guarded enqueue, alongside the KB-active
// check — tenant soft-deletion removes the workspace without touching its
// knowledge bases or durable pending ops.
type TaskPendingOpsTenantLiveness interface {
HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error)
}
// TaskPendingOpsFinalizingSeeder atomically hands a processing knowledge row
// to the asynchronous finalizing pipeline while persisting the durable
// pending operation that owns one of its subtask slots.