fix(knowledge): recover stranded deleting rows and align the KB count (#3367)
Second slice of #3338 (tracked as #3345). Count alignment: CountKnowledgeByKnowledgeBaseID now excludes rows mid-deletion, mirroring the document list's existing filter — a stranded delete no longer shows as '4 documents, 3 listed' on the KB card. Failed rows stay counted: they are visible, actionable cards. Sweep C: housekeeping now recovers rows stuck at 'deleting' whose delete task no longer exists. The delete dead-letter callback only flips rows that are still 'deleting' when retries exhaust, so a worker death mid-delete or a lost queue left the row hidden from the list forever. Same recovery contract as the parse sweep: flip to failed with a clear error so the row becomes visible and actionable; the delete executor is idempotent and plans by row ID, so a late-arriving task still finishes cleanly. Liveness gate: a new TaskInspector probe (HasQueuedDeleteTasksForKnowledge) matches knowledge:list_delete batch payloads — the per-parse matcher cannot see them. A live queued/active delete defers the row (backlog, not stranded); a probe error defers that row; a nil inspector defers the whole sweep. Lite mode's noop returns false by design: inline deletes never queue, so a stale deleting row there genuinely has no task. Tests: sweep matrix (stranded recovered / backlogged kept / fresh untouched / probe-error deferred / nil-inspector deferred), count alignment, and the payload matcher table. Refs #3345, #3338main
parent
b4aa5bbe8d
commit
c8ebb9ed96
|
|
@ -745,8 +745,12 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
|
|||
kbID string,
|
||||
) (int64, error) {
|
||||
var count int64
|
||||
// Mirror the document list's view (applyKnowledgeListFilter): rows
|
||||
// mid-deletion are hidden there, so counting them here is what produced
|
||||
// the "4 documents, 3 listed" ghost on the KB card (issues #3338/#3345).
|
||||
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?",
|
||||
tenantID, kbID, types.ParseStatusDeleting).
|
||||
Count(&count).Error
|
||||
return count, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -138,3 +138,28 @@ func TestCheckKnowledgeExists_ParseStatusMatrix(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting pins the count/list
|
||||
// alignment (issues #3338/#3345): the document list hides rows mid-deletion,
|
||||
// so the KB card count must not include them either — otherwise a stranded
|
||||
// delete shows up as "4 documents, 3 listed".
|
||||
func TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting(t *testing.T) {
|
||||
db := setupKnowledgeTestDB(t)
|
||||
repo := NewKnowledgeRepository(db)
|
||||
for _, tc := range []struct{ id, status string }{
|
||||
{"k-done", "completed"},
|
||||
{"k-failed", "failed"},
|
||||
{"k-deleting", "deleting"},
|
||||
} {
|
||||
require.NoError(t, db.Exec(`
|
||||
INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, file_name, parse_status)
|
||||
VALUES (?, 1, 'kb-cnt', 'file', ?, ?)
|
||||
`, tc.id, tc.id+".md", tc.status).Error)
|
||||
}
|
||||
|
||||
count, err := repo.CountKnowledgeByKnowledgeBaseID(context.Background(), 1, "kb-cnt")
|
||||
require.NoError(t, err)
|
||||
// failed rows stay counted (they are visible, actionable cards); only
|
||||
// the hidden mid-deletion row drops out.
|
||||
require.Equal(t, int64(2), count)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -210,6 +210,60 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
|||
} else if resSummary.RowsAffected > 0 {
|
||||
logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)
|
||||
}
|
||||
|
||||
// Sweep C: rows stuck in "deleting" whose delete task no longer exists
|
||||
// (issues #3338/#3345). The delete dead-letter callback only recovers
|
||||
// rows that are still "deleting" when retries exhaust; a worker death
|
||||
// mid-delete or a lost queue leaves the row hidden from the document
|
||||
// list forever. Same recovery contract as Sweep A: flip to failed with
|
||||
// a clear error so the row becomes visible and actionable (the user's
|
||||
// delete intent can then be retried; the delete executor is idempotent
|
||||
// and plans by row ID, so a late-arriving task still finishes cleanly).
|
||||
//
|
||||
// Liveness gate: a backlogged delete is not stranded — probe the queue
|
||||
// for a live knowledge:list_delete covering the row before touching it.
|
||||
// A nil inspector (nothing wired at all) defers every candidate to the
|
||||
// next sweep: we cannot tell backlog from orphan without it, and
|
||||
// wrongly failing a live delete is visible to users, while waiting one
|
||||
// more interval is not. A probe error defers that single row.
|
||||
if h.inspector != nil {
|
||||
var strandedDeletes []types.Knowledge
|
||||
if err := h.db.WithContext(ctx).
|
||||
Where("parse_status = ? AND updated_at < ?", types.ParseStatusDeleting, cutoff).
|
||||
Find(&strandedDeletes).Error; err != nil {
|
||||
logger.Warnf(ctx, "[Housekeeping] deleting candidate query failed: %v", err)
|
||||
} else {
|
||||
recoveredDeletes := int64(0)
|
||||
for _, k := range strandedDeletes {
|
||||
queued, err := h.inspector.HasQueuedDeleteTasksForKnowledge(ctx, k.ID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx,
|
||||
"[Housekeeping] delete-task probe failed for %s: %v (deferring to next sweep)",
|
||||
k.ID, err)
|
||||
continue
|
||||
}
|
||||
if queued {
|
||||
continue
|
||||
}
|
||||
res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("id = ? AND parse_status = ?", k.ID, types.ParseStatusDeleting).
|
||||
Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "delete task stranded (no queued/active delete task) > " + threshold.String() + ", recovered by housekeeping",
|
||||
})
|
||||
if res.Error != nil {
|
||||
logger.Warnf(ctx, "[Housekeeping] delete sweep update failed for %s: %v", k.ID, res.Error)
|
||||
continue
|
||||
}
|
||||
recoveredDeletes += res.RowsAffected
|
||||
}
|
||||
if recoveredDeletes > 0 {
|
||||
logger.Infof(ctx,
|
||||
"[Housekeeping] recovered %d stranded deleting row(s) (threshold=%s)",
|
||||
recoveredDeletes, threshold)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// filterByLastSpanActivity returns the subset of candidates whose most
|
||||
|
|
|
|||
|
|
@ -133,7 +133,10 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat
|
|||
// the probe to fail so the fail-safe branch can be exercised.
|
||||
type fakeTaskInspector struct {
|
||||
queued map[string]bool
|
||||
err error
|
||||
// deleteQueued independently controls the delete-task liveness probe;
|
||||
// nil means "no delete task alive" for every ID.
|
||||
deleteQueued map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) CancelTasksForKnowledge(
|
||||
|
|
@ -151,6 +154,15 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge(
|
|||
return f.queued[knowledgeID], nil
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) HasQueuedDeleteTasksForKnowledge(
|
||||
_ context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
if f.err != nil {
|
||||
return false, f.err
|
||||
}
|
||||
return f.deleteQueued[knowledgeID], nil
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) QueueStats(
|
||||
_ context.Context,
|
||||
) ([]types.QueueStat, bool, error) {
|
||||
|
|
@ -386,3 +398,88 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) {
|
|||
assert.Equal(t, types.ParseStatusProcessing, status,
|
||||
"knowledge updated within the cutoff must be left alone")
|
||||
}
|
||||
|
||||
// --- Sweep C: stranded "deleting" rows (issues #3338/#3345) --—
|
||||
|
||||
func readKnowledgeStatus(t *testing.T, db *gorm.DB, id string) (string, string) {
|
||||
t.Helper()
|
||||
var status, errMsg string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status, COALESCE(error_message, '') FROM knowledges WHERE id = ?`, id,
|
||||
).Row().Scan(&status, &errMsg))
|
||||
return status, errMsg
|
||||
}
|
||||
|
||||
func TestHousekeeping_RecoversStrandedDeletingRow(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
// No deleteQueued entry: the delete task is gone (worker death / lost
|
||||
// queue), the dead-letter path never fired, the row is stranded.
|
||||
svc := newHousekeepingSvcForTest(db)
|
||||
stale := time.Now().Add(-3 * time.Hour)
|
||||
insertKnowledge(t, db, "kid-stranded-delete", types.ParseStatusDeleting, stale)
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
status, errMsg := readKnowledgeStatus(t, db, "kid-stranded-delete")
|
||||
assert.Equal(t, types.ParseStatusFailed, status)
|
||||
assert.Contains(t, errMsg, "stranded")
|
||||
}
|
||||
|
||||
func TestHousekeeping_KeepsBackloggedDeletingRow(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
// A live (queued/active/retry) knowledge:list_delete still covers the
|
||||
// row — backpressure, not stranded; must stay untouched.
|
||||
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
|
||||
deleteQueued: map[string]bool{"kid-backlogged-delete": true},
|
||||
})
|
||||
insertKnowledge(t, db, "kid-backlogged-delete", types.ParseStatusDeleting,
|
||||
time.Now().Add(-3*time.Hour))
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
status, _ := readKnowledgeStatus(t, db, "kid-backlogged-delete")
|
||||
assert.Equal(t, types.ParseStatusDeleting, status)
|
||||
}
|
||||
|
||||
func TestHousekeeping_FreshDeletingRowUntouched(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
svc := newHousekeepingSvcForTest(db)
|
||||
// Marked deleting moments ago: below the staleness cutoff, so the sweep
|
||||
// must not even consider it (a delete just handed to the queue).
|
||||
insertKnowledge(t, db, "kid-fresh-delete", types.ParseStatusDeleting, time.Now())
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
status, _ := readKnowledgeStatus(t, db, "kid-fresh-delete")
|
||||
assert.Equal(t, types.ParseStatusDeleting, status)
|
||||
}
|
||||
|
||||
func TestHousekeeping_DeletingProbeErrorDefers(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
// Backend probe error must defer the row to the next sweep rather than
|
||||
// guess: failing a live delete would surface a wrong "failed" card.
|
||||
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
|
||||
err: assert.AnError,
|
||||
})
|
||||
insertKnowledge(t, db, "kid-probe-err-delete", types.ParseStatusDeleting,
|
||||
time.Now().Add(-3*time.Hour))
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
status, _ := readKnowledgeStatus(t, db, "kid-probe-err-delete")
|
||||
assert.Equal(t, types.ParseStatusDeleting, status)
|
||||
}
|
||||
|
||||
func TestHousekeeping_NilInspectorDefersDeletingSweep(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
// Without any inspector wired there is no way to tell backlog from
|
||||
// orphan, so the whole Sweep C defers — never guesses.
|
||||
svc := newHousekeepingSvcWithInspector(db, nil)
|
||||
insertKnowledge(t, db, "kid-nil-inspector", types.ParseStatusDeleting,
|
||||
time.Now().Add(-3*time.Hour))
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
status, _ := readKnowledgeStatus(t, db, "kid-nil-inspector")
|
||||
assert.Equal(t, types.ParseStatusDeleting, status)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ func (r *recordingKBTaskInspector) CancelTasksForKnowledge(
|
|||
return 0, 0, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ type runtimeTestInspector struct{}
|
|||
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -159,6 +159,28 @@ func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(
|
|||
return deleted, cancelled, nil
|
||||
}
|
||||
|
||||
// matchesKnowledgeListDelete identifies a knowledge:list_delete task whose
|
||||
// batch payload covers knowledgeID. Delete tasks carry knowledge_ids (plural)
|
||||
// and are deliberately outside taskTypesForKnowledgeCancel, so they need
|
||||
// their own matcher for the housekeeping delete sweep's liveness probe.
|
||||
func matchesKnowledgeListDelete(taskType string, payload []byte, knowledgeID string) bool {
|
||||
if taskType != types.TypeKnowledgeListDelete {
|
||||
return false
|
||||
}
|
||||
var probe struct {
|
||||
KnowledgeIDs []string `json:"knowledge_ids"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &probe); err != nil {
|
||||
return false
|
||||
}
|
||||
for _, id := range probe.KnowledgeIDs {
|
||||
if id == knowledgeID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
|
||||
// retry / active task referencing knowledgeID still lives in the queue.
|
||||
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
|
||||
|
|
@ -183,6 +205,29 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// HasQueuedDeleteTasksForKnowledge is the delete-task counterpart of
|
||||
// HasQueuedTasksForKnowledge: it matches knowledge:list_delete batch
|
||||
// payloads that still cover the knowledge ID. The housekeeping delete
|
||||
// sweep uses it to protect backlogged-but-alive deletes from recovery.
|
||||
func (a *asynqTaskInspector) HasQueuedDeleteTasksForKnowledge(
|
||||
ctx context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
if a == nil || a.inspector == nil || knowledgeID == "" {
|
||||
return false, nil
|
||||
}
|
||||
matcher := func(taskType string, payload []byte) bool {
|
||||
return matchesKnowledgeListDelete(taskType, payload, knowledgeID)
|
||||
}
|
||||
for _, queue := range queuesScanned {
|
||||
for _, state := range a.cancellableTaskStates() {
|
||||
if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// QueueStats returns a depth snapshot for every queue this app enqueues
|
||||
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
|
||||
// the result onto types.QueueStat, attaching static pool/weight metadata
|
||||
|
|
@ -1081,6 +1126,15 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge(
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// HasQueuedDeleteTasksForKnowledge always reports false in Lite mode:
|
||||
// deletes also run inline there, so a stale "deleting" row genuinely has
|
||||
// no task left and the delete sweep may recover it.
|
||||
func (noopTaskInspector) HasQueuedDeleteTasksForKnowledge(
|
||||
ctx context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// QueueStats reports "not supported" in Lite mode: there is no Redis /
|
||||
// asynq backend to inspect, so the runtime dashboard renders an
|
||||
// "unavailable in this deployment" state instead of an empty table.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import (
|
|||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsAsynqQueueNotFound(t *testing.T) {
|
||||
|
|
@ -35,3 +37,26 @@ func TestIsAsynqQueueNotFound(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesKnowledgeListDelete(t *testing.T) {
|
||||
const payload = `{"knowledge_base_id":"kb-1","tenant_id":7,"knowledge_ids":["kid-a","kid-b"]}`
|
||||
cases := []struct {
|
||||
name string
|
||||
taskType string
|
||||
payload string
|
||||
kid string
|
||||
want bool
|
||||
}{
|
||||
{"covered id", types.TypeKnowledgeListDelete, payload, "kid-a", true},
|
||||
{"last id", types.TypeKnowledgeListDelete, payload, "kid-b", true},
|
||||
{"uncovered id", types.TypeKnowledgeListDelete, payload, "kid-c", false},
|
||||
{"other task type", types.TypeDocumentProcess, payload, "kid-a", false},
|
||||
{"bad json", types.TypeKnowledgeListDelete, "{", "kid-a", false},
|
||||
{"empty list", types.TypeKnowledgeListDelete, `{"knowledge_ids":[]}`, "kid-a", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, matchesKnowledgeListDelete(tc.taskType, []byte(tc.payload), tc.kid))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,6 +44,18 @@ type TaskInspector interface {
|
|||
// span/updated_at checks remain authoritative there.
|
||||
HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
|
||||
|
||||
// HasQueuedDeleteTasksForKnowledge reports whether any pending /
|
||||
// scheduled / retry / active knowledge:list_delete task still covers
|
||||
// the given knowledge ID. The delete sweep calls it before recovering
|
||||
// a stranded "deleting" row: the delete task is enqueued with a batch
|
||||
// payload (knowledge_ids list) that the per-parse matcher above cannot
|
||||
// see, so without this probe the sweep could not tell a stranded delete
|
||||
// from one that is merely backlogged behind a busy maintenance queue.
|
||||
//
|
||||
// Same fail-safe contract: (false, err) on backend error, callers defer;
|
||||
// Lite mode always returns false — inline deletes never queue.
|
||||
HasQueuedDeleteTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
|
||||
|
||||
// QueueStats returns a read-only depth snapshot for every queue this
|
||||
// application enqueues into, for the System Admin runtime dashboard.
|
||||
//
|
||||
|
|
|
|||
Loading…
Reference in New Issue