fix(knowledge): recover stranded deleting rows and align the KB count (#3367)

Second slice of #3338 (tracked as #3345).

Count alignment: CountKnowledgeByKnowledgeBaseID now excludes rows
mid-deletion, mirroring the document list's existing filter — a stranded
delete no longer shows as '4 documents, 3 listed' on the KB card. Failed
rows stay counted: they are visible, actionable cards.

Sweep C: housekeeping now recovers rows stuck at 'deleting' whose delete
task no longer exists. The delete dead-letter callback only flips rows
that are still 'deleting' when retries exhaust, so a worker death
mid-delete or a lost queue left the row hidden from the list forever.
Same recovery contract as the parse sweep: flip to failed with a clear
error so the row becomes visible and actionable; the delete executor is
idempotent and plans by row ID, so a late-arriving task still finishes
cleanly.

Liveness gate: a new TaskInspector probe (HasQueuedDeleteTasksForKnowledge)
matches knowledge:list_delete batch payloads — the per-parse matcher
cannot see them. A live queued/active delete defers the row (backlog,
not stranded); a probe error defers that row; a nil inspector defers the
whole sweep. Lite mode's noop returns false by design: inline deletes
never queue, so a stale deleting row there genuinely has no task.

Tests: sweep matrix (stranded recovered / backlogged kept / fresh
untouched / probe-error deferred / nil-inspector deferred), count
alignment, and the payload matcher table.

Refs #3345, #3338
main
Fei.Chen 2026-09-18 00:55:44 +08:00 committed by GitHub
parent b4aa5bbe8d
commit c8ebb9ed96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 281 additions and 2 deletions

View File

@ -745,8 +745,12 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
kbID string,
) (int64, error) {
var count int64
// Mirror the document list's view (applyKnowledgeListFilter): rows
// mid-deletion are hidden there, so counting them here is what produced
// the "4 documents, 3 listed" ghost on the KB card (issues #3338/#3345).
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?",
tenantID, kbID, types.ParseStatusDeleting).
Count(&count).Error
return count, err
}

View File

@ -138,3 +138,28 @@ func TestCheckKnowledgeExists_ParseStatusMatrix(t *testing.T) {
})
}
}
// TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting pins the count/list
// alignment (issues #3338/#3345): the document list hides rows mid-deletion,
// so the KB card count must not include them either — otherwise a stranded
// delete shows up as "4 documents, 3 listed".
func TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting(t *testing.T) {
db := setupKnowledgeTestDB(t)
repo := NewKnowledgeRepository(db)
for _, tc := range []struct{ id, status string }{
{"k-done", "completed"},
{"k-failed", "failed"},
{"k-deleting", "deleting"},
} {
require.NoError(t, db.Exec(`
INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, file_name, parse_status)
VALUES (?, 1, 'kb-cnt', 'file', ?, ?)
`, tc.id, tc.id+".md", tc.status).Error)
}
count, err := repo.CountKnowledgeByKnowledgeBaseID(context.Background(), 1, "kb-cnt")
require.NoError(t, err)
// failed rows stay counted (they are visible, actionable cards); only
// the hidden mid-deletion row drops out.
require.Equal(t, int64(2), count)
}

View File

@ -210,6 +210,60 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
} else if resSummary.RowsAffected > 0 {
logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)
}
// Sweep C: rows stuck in "deleting" whose delete task no longer exists
// (issues #3338/#3345). The delete dead-letter callback only recovers
// rows that are still "deleting" when retries exhaust; a worker death
// mid-delete or a lost queue leaves the row hidden from the document
// list forever. Same recovery contract as Sweep A: flip to failed with
// a clear error so the row becomes visible and actionable (the user's
// delete intent can then be retried; the delete executor is idempotent
// and plans by row ID, so a late-arriving task still finishes cleanly).
//
// Liveness gate: a backlogged delete is not stranded — probe the queue
// for a live knowledge:list_delete covering the row before touching it.
// A nil inspector (nothing wired at all) defers every candidate to the
// next sweep: we cannot tell backlog from orphan without it, and
// wrongly failing a live delete is visible to users, while waiting one
// more interval is not. A probe error defers that single row.
if h.inspector != nil {
var strandedDeletes []types.Knowledge
if err := h.db.WithContext(ctx).
Where("parse_status = ? AND updated_at < ?", types.ParseStatusDeleting, cutoff).
Find(&strandedDeletes).Error; err != nil {
logger.Warnf(ctx, "[Housekeeping] deleting candidate query failed: %v", err)
} else {
recoveredDeletes := int64(0)
for _, k := range strandedDeletes {
queued, err := h.inspector.HasQueuedDeleteTasksForKnowledge(ctx, k.ID)
if err != nil {
logger.Warnf(ctx,
"[Housekeeping] delete-task probe failed for %s: %v (deferring to next sweep)",
k.ID, err)
continue
}
if queued {
continue
}
res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("id = ? AND parse_status = ?", k.ID, types.ParseStatusDeleting).
Updates(map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": "delete task stranded (no queued/active delete task) > " + threshold.String() + ", recovered by housekeeping",
})
if res.Error != nil {
logger.Warnf(ctx, "[Housekeeping] delete sweep update failed for %s: %v", k.ID, res.Error)
continue
}
recoveredDeletes += res.RowsAffected
}
if recoveredDeletes > 0 {
logger.Infof(ctx,
"[Housekeeping] recovered %d stranded deleting row(s) (threshold=%s)",
recoveredDeletes, threshold)
}
}
}
}
// filterByLastSpanActivity returns the subset of candidates whose most

View File

@ -133,7 +133,10 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat
// the probe to fail so the fail-safe branch can be exercised.
type fakeTaskInspector struct {
queued map[string]bool
err error
// deleteQueued independently controls the delete-task liveness probe;
// nil means "no delete task alive" for every ID.
deleteQueued map[string]bool
err error
}
func (f fakeTaskInspector) CancelTasksForKnowledge(
@ -151,6 +154,15 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge(
return f.queued[knowledgeID], nil
}
func (f fakeTaskInspector) HasQueuedDeleteTasksForKnowledge(
_ context.Context, knowledgeID string,
) (bool, error) {
if f.err != nil {
return false, f.err
}
return f.deleteQueued[knowledgeID], nil
}
func (f fakeTaskInspector) QueueStats(
_ context.Context,
) ([]types.QueueStat, bool, error) {
@ -386,3 +398,88 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) {
assert.Equal(t, types.ParseStatusProcessing, status,
"knowledge updated within the cutoff must be left alone")
}
// --- Sweep C: stranded "deleting" rows (issues #3338/#3345) --—
func readKnowledgeStatus(t *testing.T, db *gorm.DB, id string) (string, string) {
t.Helper()
var status, errMsg string
require.NoError(t, db.Raw(
`SELECT parse_status, COALESCE(error_message, '') FROM knowledges WHERE id = ?`, id,
).Row().Scan(&status, &errMsg))
return status, errMsg
}
func TestHousekeeping_RecoversStrandedDeletingRow(t *testing.T) {
db := setupHousekeepingDB(t)
// No deleteQueued entry: the delete task is gone (worker death / lost
// queue), the dead-letter path never fired, the row is stranded.
svc := newHousekeepingSvcForTest(db)
stale := time.Now().Add(-3 * time.Hour)
insertKnowledge(t, db, "kid-stranded-delete", types.ParseStatusDeleting, stale)
svc.runSweep(context.Background())
status, errMsg := readKnowledgeStatus(t, db, "kid-stranded-delete")
assert.Equal(t, types.ParseStatusFailed, status)
assert.Contains(t, errMsg, "stranded")
}
func TestHousekeeping_KeepsBackloggedDeletingRow(t *testing.T) {
db := setupHousekeepingDB(t)
// A live (queued/active/retry) knowledge:list_delete still covers the
// row — backpressure, not stranded; must stay untouched.
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
deleteQueued: map[string]bool{"kid-backlogged-delete": true},
})
insertKnowledge(t, db, "kid-backlogged-delete", types.ParseStatusDeleting,
time.Now().Add(-3*time.Hour))
svc.runSweep(context.Background())
status, _ := readKnowledgeStatus(t, db, "kid-backlogged-delete")
assert.Equal(t, types.ParseStatusDeleting, status)
}
func TestHousekeeping_FreshDeletingRowUntouched(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
// Marked deleting moments ago: below the staleness cutoff, so the sweep
// must not even consider it (a delete just handed to the queue).
insertKnowledge(t, db, "kid-fresh-delete", types.ParseStatusDeleting, time.Now())
svc.runSweep(context.Background())
status, _ := readKnowledgeStatus(t, db, "kid-fresh-delete")
assert.Equal(t, types.ParseStatusDeleting, status)
}
func TestHousekeeping_DeletingProbeErrorDefers(t *testing.T) {
db := setupHousekeepingDB(t)
// Backend probe error must defer the row to the next sweep rather than
// guess: failing a live delete would surface a wrong "failed" card.
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
err: assert.AnError,
})
insertKnowledge(t, db, "kid-probe-err-delete", types.ParseStatusDeleting,
time.Now().Add(-3*time.Hour))
svc.runSweep(context.Background())
status, _ := readKnowledgeStatus(t, db, "kid-probe-err-delete")
assert.Equal(t, types.ParseStatusDeleting, status)
}
func TestHousekeeping_NilInspectorDefersDeletingSweep(t *testing.T) {
db := setupHousekeepingDB(t)
// Without any inspector wired there is no way to tell backlog from
// orphan, so the whole Sweep C defers — never guesses.
svc := newHousekeepingSvcWithInspector(db, nil)
insertKnowledge(t, db, "kid-nil-inspector", types.ParseStatusDeleting,
time.Now().Add(-3*time.Hour))
svc.runSweep(context.Background())
status, _ := readKnowledgeStatus(t, db, "kid-nil-inspector")
assert.Equal(t, types.ParseStatusDeleting, status)
}

View File

@ -35,6 +35,10 @@ func (r *recordingKBTaskInspector) CancelTasksForKnowledge(
return 0, 0, nil
}
func (r *recordingKBTaskInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}

View File

@ -59,6 +59,10 @@ type runtimeTestInspector struct{}
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
return 0, 0, nil
}
func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}

View File

@ -159,6 +159,28 @@ func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(
return deleted, cancelled, nil
}
// matchesKnowledgeListDelete identifies a knowledge:list_delete task whose
// batch payload covers knowledgeID. Delete tasks carry knowledge_ids (plural)
// and are deliberately outside taskTypesForKnowledgeCancel, so they need
// their own matcher for the housekeeping delete sweep's liveness probe.
func matchesKnowledgeListDelete(taskType string, payload []byte, knowledgeID string) bool {
if taskType != types.TypeKnowledgeListDelete {
return false
}
var probe struct {
KnowledgeIDs []string `json:"knowledge_ids"`
}
if err := json.Unmarshal(payload, &probe); err != nil {
return false
}
for _, id := range probe.KnowledgeIDs {
if id == knowledgeID {
return true
}
}
return false
}
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
// retry / active task referencing knowledgeID still lives in the queue.
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
@ -183,6 +205,29 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
return false, nil
}
// HasQueuedDeleteTasksForKnowledge is the delete-task counterpart of
// HasQueuedTasksForKnowledge: it matches knowledge:list_delete batch
// payloads that still cover the knowledge ID. The housekeeping delete
// sweep uses it to protect backlogged-but-alive deletes from recovery.
func (a *asynqTaskInspector) HasQueuedDeleteTasksForKnowledge(
ctx context.Context, knowledgeID string,
) (bool, error) {
if a == nil || a.inspector == nil || knowledgeID == "" {
return false, nil
}
matcher := func(taskType string, payload []byte) bool {
return matchesKnowledgeListDelete(taskType, payload, knowledgeID)
}
for _, queue := range queuesScanned {
for _, state := range a.cancellableTaskStates() {
if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {
return true, nil
}
}
}
return false, nil
}
// QueueStats returns a depth snapshot for every queue this app enqueues
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
// the result onto types.QueueStat, attaching static pool/weight metadata
@ -1081,6 +1126,15 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge(
return false, nil
}
// HasQueuedDeleteTasksForKnowledge always reports false in Lite mode:
// deletes also run inline there, so a stale "deleting" row genuinely has
// no task left and the delete sweep may recover it.
func (noopTaskInspector) HasQueuedDeleteTasksForKnowledge(
ctx context.Context, knowledgeID string,
) (bool, error) {
return false, nil
}
// QueueStats reports "not supported" in Lite mode: there is no Redis /
// asynq backend to inspect, so the runtime dashboard renders an
// "unavailable in this deployment" state instead of an empty table.

View File

@ -5,7 +5,9 @@ import (
"fmt"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/require"
)
func TestIsAsynqQueueNotFound(t *testing.T) {
@ -35,3 +37,26 @@ func TestIsAsynqQueueNotFound(t *testing.T) {
})
}
}
func TestMatchesKnowledgeListDelete(t *testing.T) {
const payload = `{"knowledge_base_id":"kb-1","tenant_id":7,"knowledge_ids":["kid-a","kid-b"]}`
cases := []struct {
name string
taskType string
payload string
kid string
want bool
}{
{"covered id", types.TypeKnowledgeListDelete, payload, "kid-a", true},
{"last id", types.TypeKnowledgeListDelete, payload, "kid-b", true},
{"uncovered id", types.TypeKnowledgeListDelete, payload, "kid-c", false},
{"other task type", types.TypeDocumentProcess, payload, "kid-a", false},
{"bad json", types.TypeKnowledgeListDelete, "{", "kid-a", false},
{"empty list", types.TypeKnowledgeListDelete, `{"knowledge_ids":[]}`, "kid-a", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, matchesKnowledgeListDelete(tc.taskType, []byte(tc.payload), tc.kid))
})
}
}

View File

@ -44,6 +44,18 @@ type TaskInspector interface {
// span/updated_at checks remain authoritative there.
HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
// HasQueuedDeleteTasksForKnowledge reports whether any pending /
// scheduled / retry / active knowledge:list_delete task still covers
// the given knowledge ID. The delete sweep calls it before recovering
// a stranded "deleting" row: the delete task is enqueued with a batch
// payload (knowledge_ids list) that the per-parse matcher above cannot
// see, so without this probe the sweep could not tell a stranded delete
// from one that is merely backlogged behind a busy maintenance queue.
//
// Same fail-safe contract: (false, err) on backend error, callers defer;
// Lite mode always returns false — inline deletes never queue.
HasQueuedDeleteTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
// QueueStats returns a read-only depth snapshot for every queue this
// application enqueues into, for the System Admin runtime dashboard.
//