diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index 1823732c..f35b7212 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -53,11 +53,13 @@ func preparePendingOp(op *types.TaskPendingOp) error { } // EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new -// durable work after a KB was soft-deleted. On Postgres the share lock -// serializes this check+insert transaction against the row update performed by -// soft deletion: whichever operation acquires the row first determines the -// order, and the deletion path's subsequent scope scrub removes any insert -// that committed before it. +// durable work after a KB was soft-deleted. The tenant must also be alive: a +// tenant soft-deletion removes the workspace without touching its knowledge +// bases, and a deleted tenant must never accrue new model-backed work. On +// Postgres the share lock serializes this check+insert transaction against +// the row update performed by soft deletion: whichever operation acquires the +// row first determines the order, and the deletion path's subsequent scope +// scrub removes any insert that committed before it. func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( ctx context.Context, op *types.TaskPendingOp, @@ -84,6 +86,10 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( } return err } + active, err := tenantActiveWithinTx(tx, op.TenantID) + if err != nil || !active { + return err + } if err := tx.Create(op).Error; err != nil { return err } @@ -93,6 +99,31 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( return accepted, err } +// tenantActiveWithinTx reports whether the tenant row exists and is not +// soft-deleted, inside the caller's transaction. Callers that cannot see a +// tenants table (legacy test doubles) fail closed. +func tenantActiveWithinTx(tx *gorm.DB, tenantID uint64) (bool, error) { + if tenantID == 0 { + return false, nil + } + var tenant types.Tenant + err := tx.Model(&types.Tenant{}).Select("id").Where("id = ?", tenantID).Take(&tenant).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// HasActiveTenant reports whether the tenant exists and has not been +// soft-deleted. Wiki task consumers call this before doing durable or +// model-backed work on the tenant's behalf (#3593). +func (r *taskPendingOpsRepository) HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) { + return tenantActiveWithinTx(r.db.WithContext(ctx), tenantID) +} + // SeedKnowledgeFinalizingWithPendingOp commits the finalizing counter and the // durable operation that owns one slot in the same transaction. This closes // the crash window where a knowledge row could enter finalizing before its diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index 87d9bf18..dc8fe547 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -392,15 +392,32 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) { deleted_at DATETIME )`).Error) require.NoError(t, db.Exec( - "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)", - "kb-active", 1, "kb-deleted", 1, time.Now(), + "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?), (?, ?, NULL)", + "kb-active", 1, "kb-deleted", 1, time.Now(), "kb-t2", 2, + ).Error) + require.NoError(t, db.Exec(`CREATE TABLE tenants ( + id INTEGER PRIMARY KEY, + deleted_at DATETIME + )`).Error) + require.NoError(t, db.Exec( + "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, ?)", + 1, 2, time.Now(), ).Error) repo := NewTaskPendingOpsRepository(db) guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard) require.True(t, ok) + liveness, ok := repo.(interfaces.TaskPendingOpsTenantLiveness) + require.True(t, ok, "task pending repository must expose tenant liveness for wiki task guards") ctx := context.Background() + activeTenant, err := liveness.HasActiveTenant(ctx, 1) + require.NoError(t, err) + assert.True(t, activeTenant) + deletedTenant, err := liveness.HasActiveTenant(ctx, 2) + require.NoError(t, err) + assert.False(t, deletedTenant) + accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil)) require.NoError(t, err) @@ -426,6 +443,10 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) { TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase, ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant", }}, + {name: "deleted tenant with live KB", op: &types.TaskPendingOp{ + TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase, + ScopeID: "kb-t2", Op: "ingest", DedupKey: "deleted-tenant", + }}, } { t.Run(tc.name, func(t *testing.T) { accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op) diff --git a/internal/application/service/wiki_deleted_tenant_guard_test.go b/internal/application/service/wiki_deleted_tenant_guard_test.go new file mode 100644 index 00000000..d84c6944 --- /dev/null +++ b/internal/application/service/wiki_deleted_tenant_guard_test.go @@ -0,0 +1,93 @@ +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wikiTenantGuardPendingRepo stubs the slice of TaskPendingOpsRepository the +// tenant-liveness guard touches: the liveness lookup and the scope cleanup. +type wikiTenantGuardPendingRepo struct { + interfaces.TaskPendingOpsRepository + active bool + lookupErr error + lookups []uint64 + deletedKBs []string +} + +func (r *wikiTenantGuardPendingRepo) HasActiveTenant(_ context.Context, tenantID uint64) (bool, error) { + r.lookups = append(r.lookups, tenantID) + return r.active, r.lookupErr +} + +func (r *wikiTenantGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error { + if scope == types.TaskScopeKnowledgeBase { + r.deletedKBs = append(r.deletedKBs, scopeID) + } + return nil +} + +func TestWikiTaskSkipsDeletedTenant(t *testing.T) { + payload, err := json.Marshal(WikiIngestPayload{ + TenantID: 7, + KnowledgeBaseID: "kb-orphaned", + Language: "en", + }) + require.NoError(t, err) + + t.Run("ingest clears queue and does not proceed", func(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload)) + require.NoError(t, err) + require.Len(t, repo.lookups, 1) + assert.Equal(t, uint64(7), repo.lookups[0]) + assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs) + }) + + t.Run("finalize clears queue and does not proceed", func(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.ProcessWikiFinalize(context.Background(), asynq.NewTask(types.TypeWikiFinalize, payload)) + require.NoError(t, err) + require.Len(t, repo.lookups, 1) + assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs) + }) +} + +func TestWikiTenantGuardFailsOpenOnLookupError(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false, lookupErr: assert.AnError} + svc := &wikiIngestService{pendingRepo: repo} + + // A transient liveness-lookup failure must not be read as "deleted": + // the task keeps its retry budget and the enqueue guard / startup + // recovery enforce the same invariant on their own DB access. + assert.False(t, svc.tenantIsDeleted(context.Background(), 7)) + assert.Empty(t, repo.deletedKBs) +} + +func TestWikiTenantGuardWithoutLivenessCapability(t *testing.T) { + // A pending repo without the liveness extension (legacy test doubles, + // alternate implementations) is never treated as a deleted tenant. + svc := &wikiIngestService{pendingRepo: &wikiKBGuardPendingRepo{}} + + assert.False(t, svc.tenantIsDeleted(context.Background(), 7)) +} + +func TestWikiTenantGuardIgnoresZeroTenant(t *testing.T) { + repo := &wikiTenantGuardPendingRepo{active: false} + svc := &wikiIngestService{pendingRepo: repo} + + assert.False(t, svc.tenantIsDeleted(context.Background(), 0)) + assert.Empty(t, repo.lookups) + _ = interfaces.TaskPendingOpsTenantLiveness(nil) +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index da081a5a..097d85f8 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -720,6 +720,31 @@ func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Cont return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID) } +// tenantIsDeleted reports whether the payload's tenant has been soft-deleted. +// A tenant deletion removes the workspace but leaves its knowledge bases and +// durable pending ops in place, so wiki tasks restored from those ops would +// otherwise keep issuing model requests for a tenant nobody can reach (#3593). +// +// Fail-open on transient lookup errors and when the pending repo does not +// expose tenant liveness (legacy test doubles): the retry machinery still +// covers the task, and the guarded enqueue / startup recovery paths enforce +// the same invariant on their own DB access. +func (s *wikiIngestService) tenantIsDeleted(ctx context.Context, tenantID uint64) bool { + if tenantID == 0 { + return false + } + checker, ok := s.pendingRepo.(interfaces.TaskPendingOpsTenantLiveness) + if !ok { + return false + } + active, err := checker.HasActiveTenant(ctx, tenantID) + if err != nil { + logger.Warnf(ctx, "wiki: tenant liveness lookup failed for tenant %d: %v (failing open)", tenantID, err) + return false + } + return !active +} + // releaseIngestForUnavailableWiki drops the KB's queued ingest ops when the // wiki cannot run for a reason retries will not fix, and releases each // document's wiki slot in the same transaction so it leaves "finalizing". diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index d2d79414..f1d92ad5 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -254,6 +254,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } + // A soft-deleted tenant owns no reachable workspace anymore; its KBs and + // durable pending ops survive the deletion, so without this guard the + // batch would keep issuing model requests (and finalize would rebuild + // index pages) for a tenant nobody can see (#3593). Drop the KB's queue. + if s.tenantIsDeleted(ctx, payload.TenantID) { + exitStatus = "tenant_deleted" + if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil { + return fmt.Errorf("wiki ingest: clear deleted tenant queue: %w", err) + } + return nil + } + // Concurrency model (Phase 3): // // - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches @@ -931,6 +943,16 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta return nil } + // Same tenant-liveness guard as the ingest batch: finalize calls the + // synthesis model to rebuild the index page, and a deleted tenant must + // never accrue new model requests (#3593). Drop the KB's queue. + if s.tenantIsDeleted(ctx, payload.TenantID) { + if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil { + return fmt.Errorf("wiki finalize: clear deleted tenant queue: %w", err) + } + return nil + } + // Per-KB finalize lock, separate from the ingest active lock so finalize // and ingest batches never block each other. Coalescing via asynq.TaskID // already keeps at most one finalize pending per KB; this guards the diff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.go index 1867e055..adf24202 100644 --- a/internal/container/recover_pending_wiki_tasks.go +++ b/internal/container/recover_pending_wiki_tasks.go @@ -34,16 +34,24 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { return } ctx := context.Background() - const activeKnowledgeBase = `EXISTS ( + // The outer parentheses matter: the cleanup below negates this whole + // predicate with NOT, and `NOT EXISTS(...) AND EXISTS(...)` would parse + // as `(NOT EXISTS(...)) AND EXISTS(...)` — silently matching nothing. + const activeKnowledgeBase = `(EXISTS ( SELECT 1 FROM knowledge_bases kb WHERE kb.id = task_pending_ops.scope_id AND kb.tenant_id = task_pending_ops.tenant_id AND kb.deleted_at IS NULL - )` + ) AND EXISTS ( + SELECT 1 FROM tenants t + WHERE t.id = task_pending_ops.tenant_id + AND t.deleted_at IS NULL + ))` wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize} - // Durable rows for a deleted/missing KB must not recreate ephemeral - // triggers at startup. Fail closed if this cleanup cannot be verified. + // Durable rows for a deleted/missing KB — or a soft-deleted tenant — + // must not recreate ephemeral triggers at startup. Fail closed if this + // cleanup cannot be verified. cleanup := db.WithContext(ctx). Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes). Where("NOT " + activeKnowledgeBase). @@ -53,7 +61,9 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { return } if cleanup.RowsAffected > 0 { - logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected) + logger.Infof(ctx, + "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases or tenants", + cleanup.RowsAffected) } var scopes []pendingWikiScope diff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.go index 47d4c024..4df02aa0 100644 --- a/internal/container/reset_pending_tasks_test.go +++ b/internal/container/reset_pending_tasks_test.go @@ -89,6 +89,13 @@ CREATE TABLE IF NOT EXISTS knowledge_bases ( ); ` +var resetPendingTenantsDDL = ` +CREATE TABLE IF NOT EXISTS tenants ( + id INTEGER PRIMARY KEY, + deleted_at DATETIME +); +` + func setupResetPendingDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) @@ -98,6 +105,14 @@ func setupResetPendingDB(t *testing.T) *gorm.DB { require.NoError(t, db.Exec(resetPendingSpansDDL).Error) require.NoError(t, db.Exec(resetPendingOpsDDL).Error) require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error) + require.NoError(t, db.Exec(resetPendingTenantsDDL).Error) + // Tenants 7 and 8 are alive; tenant 9 is soft-deleted — its KB-scoped + // pending rows must be purged by recovery even though the KB row is + // still present (#3593). + require.NoError(t, db.Exec( + "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, NULL), (?, ?)", + 7, 8, 9, time.Now(), + ).Error) return db } @@ -309,8 +324,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { db := setupResetPendingDB(t) require.NoError(t, db.Exec( `INSERT INTO knowledge_bases (id, tenant_id, deleted_at) - VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`, - "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), + VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?), (?, ?, NULL)`, + "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), "kb-t9", 9, ).Error) rows := []struct { tenantID uint64 @@ -323,6 +338,7 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { {7, types.TypeWikiFinalize, "kb-a", "slug-a"}, {8, types.TypeWikiIngest, "kb-b", "k-3"}, {9, types.TypeWikiIngest, "kb-deleted", "k-deleted"}, + {9, types.TypeWikiIngest, "kb-t9", "k-t9"}, // live KB of a deleted tenant {10, types.TypeWikiFinalize, "kb-missing", "k-missing"}, } for _, row := range rows { @@ -350,7 +366,14 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { var orphaned int64 require.NoError(t, db.Model(&types.TaskPendingOp{}). - Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}). + Where("scope_id IN ?", []string{"kb-deleted", "kb-missing", "kb-t9"}). Count(&orphaned).Error) assert.Zero(t, orphaned) + + // The live KB of a soft-deleted tenant must be purged too: the KB row + // survives the tenant deletion, so the KB-only predicate would keep + // re-arming its triggers at every startup (#3593). + var remaining int64 + require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&remaining).Error) + assert.Equal(t, int64(4), remaining) } diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index c2fae3f0..e6d8dfd7 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -114,6 +114,16 @@ type TaskPendingOpsKnowledgeBaseGuard interface { EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error) } +// TaskPendingOpsTenantLiveness reports whether a tenant is still alive (not +// soft-deleted). Wiki task consumers use it to guarantee a deleted tenant +// never triggers new model requests: the check runs at task entry (ingest +// and finalize) and inside the guarded enqueue, alongside the KB-active +// check — tenant soft-deletion removes the workspace without touching its +// knowledge bases or durable pending ops. +type TaskPendingOpsTenantLiveness interface { + HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) +} + // TaskPendingOpsFinalizingSeeder atomically hands a processing knowledge row // to the asynchronous finalizing pipeline while persisting the durable // pending operation that owns one of its subtask slots.