From c8ebb9ed96fafbb02b8e933910579404ffad1e73 Mon Sep 17 00:00:00 2001 From: "Fei.Chen" Date: Fri, 18 Sep 2026 00:55:44 +0800 Subject: [PATCH] fix(knowledge): recover stranded deleting rows and align the KB count (#3367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second slice of #3338 (tracked as #3345). Count alignment: CountKnowledgeByKnowledgeBaseID now excludes rows mid-deletion, mirroring the document list's existing filter — a stranded delete no longer shows as '4 documents, 3 listed' on the KB card. Failed rows stay counted: they are visible, actionable cards. Sweep C: housekeeping now recovers rows stuck at 'deleting' whose delete task no longer exists. The delete dead-letter callback only flips rows that are still 'deleting' when retries exhaust, so a worker death mid-delete or a lost queue left the row hidden from the list forever. Same recovery contract as the parse sweep: flip to failed with a clear error so the row becomes visible and actionable; the delete executor is idempotent and plans by row ID, so a late-arriving task still finishes cleanly. Liveness gate: a new TaskInspector probe (HasQueuedDeleteTasksForKnowledge) matches knowledge:list_delete batch payloads — the per-parse matcher cannot see them. A live queued/active delete defers the row (backlog, not stranded); a probe error defers that row; a nil inspector defers the whole sweep. Lite mode's noop returns false by design: inline deletes never queue, so a stale deleting row there genuinely has no task. Tests: sweep matrix (stranded recovered / backlogged kept / fresh untouched / probe-error deferred / nil-inspector deferred), count alignment, and the payload matcher table. Refs #3345, #3338 --- internal/application/repository/knowledge.go | 6 +- .../repository/knowledge_duplicate_test.go | 25 +++++ .../service/knowledge_housekeeping.go | 54 ++++++++++ .../service/knowledge_housekeeping_test.go | 99 ++++++++++++++++++- .../service/knowledgebase_task_cancel_test.go | 4 + internal/handler/system_runtime_test.go | 4 + internal/router/task_inspector.go | 54 ++++++++++ internal/router/task_inspector_errors_test.go | 25 +++++ internal/types/interfaces/task_inspector.go | 12 +++ 9 files changed, 281 insertions(+), 2 deletions(-) diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 030e1fc9..4b29c552 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -745,8 +745,12 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID( kbID string, ) (int64, error) { var count int64 + // Mirror the document list's view (applyKnowledgeListFilter): rows + // mid-deletion are hidden there, so counting them here is what produced + // the "4 documents, 3 listed" ghost on the KB card (issues #3338/#3345). err := r.db.WithContext(ctx).Model(&types.Knowledge{}). - Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID). + Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", + tenantID, kbID, types.ParseStatusDeleting). Count(&count).Error return count, err } diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.go index 32f18800..56256055 100644 --- a/internal/application/repository/knowledge_duplicate_test.go +++ b/internal/application/repository/knowledge_duplicate_test.go @@ -138,3 +138,28 @@ func TestCheckKnowledgeExists_ParseStatusMatrix(t *testing.T) { }) } } + +// TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting pins the count/list +// alignment (issues #3338/#3345): the document list hides rows mid-deletion, +// so the KB card count must not include them either — otherwise a stranded +// delete shows up as "4 documents, 3 listed". +func TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db) + for _, tc := range []struct{ id, status string }{ + {"k-done", "completed"}, + {"k-failed", "failed"}, + {"k-deleting", "deleting"}, + } { + require.NoError(t, db.Exec(` + INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, file_name, parse_status) + VALUES (?, 1, 'kb-cnt', 'file', ?, ?) + `, tc.id, tc.id+".md", tc.status).Error) + } + + count, err := repo.CountKnowledgeByKnowledgeBaseID(context.Background(), 1, "kb-cnt") + require.NoError(t, err) + // failed rows stay counted (they are visible, actionable cards); only + // the hidden mid-deletion row drops out. + require.Equal(t, int64(2), count) +} diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index 4b4b3a96..57b9cf7f 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -210,6 +210,60 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { } else if resSummary.RowsAffected > 0 { logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected) } + + // Sweep C: rows stuck in "deleting" whose delete task no longer exists + // (issues #3338/#3345). The delete dead-letter callback only recovers + // rows that are still "deleting" when retries exhaust; a worker death + // mid-delete or a lost queue leaves the row hidden from the document + // list forever. Same recovery contract as Sweep A: flip to failed with + // a clear error so the row becomes visible and actionable (the user's + // delete intent can then be retried; the delete executor is idempotent + // and plans by row ID, so a late-arriving task still finishes cleanly). + // + // Liveness gate: a backlogged delete is not stranded — probe the queue + // for a live knowledge:list_delete covering the row before touching it. + // A nil inspector (nothing wired at all) defers every candidate to the + // next sweep: we cannot tell backlog from orphan without it, and + // wrongly failing a live delete is visible to users, while waiting one + // more interval is not. A probe error defers that single row. + if h.inspector != nil { + var strandedDeletes []types.Knowledge + if err := h.db.WithContext(ctx). + Where("parse_status = ? AND updated_at < ?", types.ParseStatusDeleting, cutoff). + Find(&strandedDeletes).Error; err != nil { + logger.Warnf(ctx, "[Housekeeping] deleting candidate query failed: %v", err) + } else { + recoveredDeletes := int64(0) + for _, k := range strandedDeletes { + queued, err := h.inspector.HasQueuedDeleteTasksForKnowledge(ctx, k.ID) + if err != nil { + logger.Warnf(ctx, + "[Housekeeping] delete-task probe failed for %s: %v (deferring to next sweep)", + k.ID, err) + continue + } + if queued { + continue + } + res := h.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("id = ? AND parse_status = ?", k.ID, types.ParseStatusDeleting). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": "delete task stranded (no queued/active delete task) > " + threshold.String() + ", recovered by housekeeping", + }) + if res.Error != nil { + logger.Warnf(ctx, "[Housekeeping] delete sweep update failed for %s: %v", k.ID, res.Error) + continue + } + recoveredDeletes += res.RowsAffected + } + if recoveredDeletes > 0 { + logger.Infof(ctx, + "[Housekeeping] recovered %d stranded deleting row(s) (threshold=%s)", + recoveredDeletes, threshold) + } + } + } } // filterByLastSpanActivity returns the subset of candidates whose most diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index f0491c60..c0f1938a 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -133,7 +133,10 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat // the probe to fail so the fail-safe branch can be exercised. type fakeTaskInspector struct { queued map[string]bool - err error + // deleteQueued independently controls the delete-task liveness probe; + // nil means "no delete task alive" for every ID. + deleteQueued map[string]bool + err error } func (f fakeTaskInspector) CancelTasksForKnowledge( @@ -151,6 +154,15 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge( return f.queued[knowledgeID], nil } +func (f fakeTaskInspector) HasQueuedDeleteTasksForKnowledge( + _ context.Context, knowledgeID string, +) (bool, error) { + if f.err != nil { + return false, f.err + } + return f.deleteQueued[knowledgeID], nil +} + func (f fakeTaskInspector) QueueStats( _ context.Context, ) ([]types.QueueStat, bool, error) { @@ -386,3 +398,88 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) { assert.Equal(t, types.ParseStatusProcessing, status, "knowledge updated within the cutoff must be left alone") } + +// --- Sweep C: stranded "deleting" rows (issues #3338/#3345) --— + +func readKnowledgeStatus(t *testing.T, db *gorm.DB, id string) (string, string) { + t.Helper() + var status, errMsg string + require.NoError(t, db.Raw( + `SELECT parse_status, COALESCE(error_message, '') FROM knowledges WHERE id = ?`, id, + ).Row().Scan(&status, &errMsg)) + return status, errMsg +} + +func TestHousekeeping_RecoversStrandedDeletingRow(t *testing.T) { + db := setupHousekeepingDB(t) + // No deleteQueued entry: the delete task is gone (worker death / lost + // queue), the dead-letter path never fired, the row is stranded. + svc := newHousekeepingSvcForTest(db) + stale := time.Now().Add(-3 * time.Hour) + insertKnowledge(t, db, "kid-stranded-delete", types.ParseStatusDeleting, stale) + + svc.runSweep(context.Background()) + + status, errMsg := readKnowledgeStatus(t, db, "kid-stranded-delete") + assert.Equal(t, types.ParseStatusFailed, status) + assert.Contains(t, errMsg, "stranded") +} + +func TestHousekeeping_KeepsBackloggedDeletingRow(t *testing.T) { + db := setupHousekeepingDB(t) + // A live (queued/active/retry) knowledge:list_delete still covers the + // row — backpressure, not stranded; must stay untouched. + svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{ + deleteQueued: map[string]bool{"kid-backlogged-delete": true}, + }) + insertKnowledge(t, db, "kid-backlogged-delete", types.ParseStatusDeleting, + time.Now().Add(-3*time.Hour)) + + svc.runSweep(context.Background()) + + status, _ := readKnowledgeStatus(t, db, "kid-backlogged-delete") + assert.Equal(t, types.ParseStatusDeleting, status) +} + +func TestHousekeeping_FreshDeletingRowUntouched(t *testing.T) { + db := setupHousekeepingDB(t) + svc := newHousekeepingSvcForTest(db) + // Marked deleting moments ago: below the staleness cutoff, so the sweep + // must not even consider it (a delete just handed to the queue). + insertKnowledge(t, db, "kid-fresh-delete", types.ParseStatusDeleting, time.Now()) + + svc.runSweep(context.Background()) + + status, _ := readKnowledgeStatus(t, db, "kid-fresh-delete") + assert.Equal(t, types.ParseStatusDeleting, status) +} + +func TestHousekeeping_DeletingProbeErrorDefers(t *testing.T) { + db := setupHousekeepingDB(t) + // Backend probe error must defer the row to the next sweep rather than + // guess: failing a live delete would surface a wrong "failed" card. + svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{ + err: assert.AnError, + }) + insertKnowledge(t, db, "kid-probe-err-delete", types.ParseStatusDeleting, + time.Now().Add(-3*time.Hour)) + + svc.runSweep(context.Background()) + + status, _ := readKnowledgeStatus(t, db, "kid-probe-err-delete") + assert.Equal(t, types.ParseStatusDeleting, status) +} + +func TestHousekeeping_NilInspectorDefersDeletingSweep(t *testing.T) { + db := setupHousekeepingDB(t) + // Without any inspector wired there is no way to tell backlog from + // orphan, so the whole Sweep C defers — never guesses. + svc := newHousekeepingSvcWithInspector(db, nil) + insertKnowledge(t, db, "kid-nil-inspector", types.ParseStatusDeleting, + time.Now().Add(-3*time.Hour)) + + svc.runSweep(context.Background()) + + status, _ := readKnowledgeStatus(t, db, "kid-nil-inspector") + assert.Equal(t, types.ParseStatusDeleting, status) +} diff --git a/internal/application/service/knowledgebase_task_cancel_test.go b/internal/application/service/knowledgebase_task_cancel_test.go index 250b8607..30d77536 100644 --- a/internal/application/service/knowledgebase_task_cancel_test.go +++ b/internal/application/service/knowledgebase_task_cancel_test.go @@ -35,6 +35,10 @@ func (r *recordingKBTaskInspector) CancelTasksForKnowledge( return 0, 0, nil } +func (r *recordingKBTaskInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) { + return false, nil +} + func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) { return false, nil } diff --git a/internal/handler/system_runtime_test.go b/internal/handler/system_runtime_test.go index c7e3c3c8..3dfe7d5d 100644 --- a/internal/handler/system_runtime_test.go +++ b/internal/handler/system_runtime_test.go @@ -59,6 +59,10 @@ type runtimeTestInspector struct{} func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) { return 0, 0, nil } +func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) { + return false, nil +} + func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) { return false, nil } diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index bdef9a30..b4404c5d 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -159,6 +159,28 @@ func (a *asynqTaskInspector) CancelTasksForKnowledgeBase( return deleted, cancelled, nil } +// matchesKnowledgeListDelete identifies a knowledge:list_delete task whose +// batch payload covers knowledgeID. Delete tasks carry knowledge_ids (plural) +// and are deliberately outside taskTypesForKnowledgeCancel, so they need +// their own matcher for the housekeeping delete sweep's liveness probe. +func matchesKnowledgeListDelete(taskType string, payload []byte, knowledgeID string) bool { + if taskType != types.TypeKnowledgeListDelete { + return false + } + var probe struct { + KnowledgeIDs []string `json:"knowledge_ids"` + } + if err := json.Unmarshal(payload, &probe); err != nil { + return false + } + for _, id := range probe.KnowledgeIDs { + if id == knowledgeID { + return true + } + } + return false +} + // HasQueuedTasksForKnowledge reports whether any pending / scheduled / // retry / active task referencing knowledgeID still lives in the queue. // Read-only counterpart of CancelTasksForKnowledge — the housekeeping @@ -183,6 +205,29 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( return false, nil } +// HasQueuedDeleteTasksForKnowledge is the delete-task counterpart of +// HasQueuedTasksForKnowledge: it matches knowledge:list_delete batch +// payloads that still cover the knowledge ID. The housekeeping delete +// sweep uses it to protect backlogged-but-alive deletes from recovery. +func (a *asynqTaskInspector) HasQueuedDeleteTasksForKnowledge( + ctx context.Context, knowledgeID string, +) (bool, error) { + if a == nil || a.inspector == nil || knowledgeID == "" { + return false, nil + } + matcher := func(taskType string, payload []byte) bool { + return matchesKnowledgeListDelete(taskType, payload, knowledgeID) + } + for _, queue := range queuesScanned { + for _, state := range a.cancellableTaskStates() { + if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) { + return true, nil + } + } + } + return false, nil +} + // QueueStats returns a depth snapshot for every queue this app enqueues // into. Read-only: it calls Inspector.GetQueueInfo per queue and maps // the result onto types.QueueStat, attaching static pool/weight metadata @@ -1081,6 +1126,15 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge( return false, nil } +// HasQueuedDeleteTasksForKnowledge always reports false in Lite mode: +// deletes also run inline there, so a stale "deleting" row genuinely has +// no task left and the delete sweep may recover it. +func (noopTaskInspector) HasQueuedDeleteTasksForKnowledge( + ctx context.Context, knowledgeID string, +) (bool, error) { + return false, nil +} + // QueueStats reports "not supported" in Lite mode: there is no Redis / // asynq backend to inspect, so the runtime dashboard renders an // "unavailable in this deployment" state instead of an empty table. diff --git a/internal/router/task_inspector_errors_test.go b/internal/router/task_inspector_errors_test.go index 2f93356a..59ca9cc1 100644 --- a/internal/router/task_inspector_errors_test.go +++ b/internal/router/task_inspector_errors_test.go @@ -5,7 +5,9 @@ import ( "fmt" "testing" + "github.com/Tencent/WeKnora/internal/types" "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" ) func TestIsAsynqQueueNotFound(t *testing.T) { @@ -35,3 +37,26 @@ func TestIsAsynqQueueNotFound(t *testing.T) { }) } } + +func TestMatchesKnowledgeListDelete(t *testing.T) { + const payload = `{"knowledge_base_id":"kb-1","tenant_id":7,"knowledge_ids":["kid-a","kid-b"]}` + cases := []struct { + name string + taskType string + payload string + kid string + want bool + }{ + {"covered id", types.TypeKnowledgeListDelete, payload, "kid-a", true}, + {"last id", types.TypeKnowledgeListDelete, payload, "kid-b", true}, + {"uncovered id", types.TypeKnowledgeListDelete, payload, "kid-c", false}, + {"other task type", types.TypeDocumentProcess, payload, "kid-a", false}, + {"bad json", types.TypeKnowledgeListDelete, "{", "kid-a", false}, + {"empty list", types.TypeKnowledgeListDelete, `{"knowledge_ids":[]}`, "kid-a", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, matchesKnowledgeListDelete(tc.taskType, []byte(tc.payload), tc.kid)) + }) + } +} diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go index 593c7fe4..6badea74 100644 --- a/internal/types/interfaces/task_inspector.go +++ b/internal/types/interfaces/task_inspector.go @@ -44,6 +44,18 @@ type TaskInspector interface { // span/updated_at checks remain authoritative there. HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error) + // HasQueuedDeleteTasksForKnowledge reports whether any pending / + // scheduled / retry / active knowledge:list_delete task still covers + // the given knowledge ID. The delete sweep calls it before recovering + // a stranded "deleting" row: the delete task is enqueued with a batch + // payload (knowledge_ids list) that the per-parse matcher above cannot + // see, so without this probe the sweep could not tell a stranded delete + // from one that is merely backlogged behind a busy maintenance queue. + // + // Same fail-safe contract: (false, err) on backend error, callers defer; + // Lite mode always returns false — inline deletes never queue. + HasQueuedDeleteTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error) + // QueueStats returns a read-only depth snapshot for every queue this // application enqueues into, for the System Admin runtime dashboard. //