From 5780affdfc76342ddd0f5cf95b548a1a4d0b2a5a Mon Sep 17 00:00:00 2001 From: Myang_print <1182470296@qq.com> Date: Fri, 31 Jul 2026 12:55:50 +0800 Subject: [PATCH] fix(knowledge): surface batch reparse failures Signed-off-by: Myang_print <1182470296@qq.com> --- docs/api/knowledge.md | 45 ++++++ .../service/knowledge_batch_reparse_test.go | 136 ++++++++++++++++++ .../application/service/knowledge_process.go | 82 ++++++++--- 3 files changed, 240 insertions(+), 23 deletions(-) create mode 100644 internal/application/service/knowledge_batch_reparse_test.go diff --git a/docs/api/knowledge.md b/docs/api/knowledge.md index 065a9164..d4c632e3 100644 --- a/docs/api/knowledge.md +++ b/docs/api/knowledge.md @@ -23,6 +23,7 @@ | PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 | | PUT | `/knowledge/tags` | 批量更新知识标签 | | GET | `/knowledge/search` | 跨知识库搜索/过滤知识 | +| POST | `/knowledge/batch-reparse` | 同一知识库内批量重新解析知识(异步任务) | | POST | `/knowledge/batch-delete` | 同一知识库内批量删除知识(异步任务) | | POST | `/knowledge/move` | 迁移知识到另一知识库(异步任务) | | GET | `/knowledge/move/progress/:task_id` | 查询知识迁移任务进度 | @@ -782,6 +783,50 @@ curl --location --get 'http://localhost:8080/api/v1/knowledge/search' \ `agent_id=...&` 且该 Agent 的 KB 选择模式为 `none` 时,将直接返回 `data: []` 与 `has_more: false`。 +## POST `/knowledge/batch-reparse` - 同一知识库内批量重新解析 + +按 ID 列表在单个知识库内批量重新解析知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 存在并属于同一 `kb_id`。 + +**请求体**: + +| 字段 | 类型 | 必填 | 说明 | +| ---------------- | -------- | ---- | ------------------------------------------------ | +| `kb_id` | string | 是 | 目标知识库 ID | +| `ids` | string[] | 是 | 待重新解析的知识 ID 列表(≤ 200) | +| `process_config` | object | 否 | 本批次共用的处理配置覆盖;省略时沿用各知识原配置 | + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge/batch-reparse' \ +--header 'X-API-Key: sk-xxxxx' \ +--header 'Content-Type: application/json' \ +--data '{ + "kb_id": "kb-00000001", + "ids": [ + "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "9c8af585-ae15-44ce-8f73-45ad18394651" + ] +}' +``` + +**响应**: + +```json +{ + "success": true, + "message": "Batch reparse task submitted", + "data": { + "task_id": "a1b2c3d4", + "reparse_count": 2 + } +} +``` + +HTTP 成功表示批处理包装任务已入队。后台会尝试提交列表中的每个知识;单项提交失败时,该知识会进入 `failed` 状态并保留错误信息,批处理任务也会在运行队列中标记失败。为避免重复清理已成功提交的知识,出现部分失败后不会自动重试整个批次;可筛选失败知识后再次批量重新解析。 + +任一 ID 不属于 `kb_id` 或不存在时,返回 400 并整批拒绝。 + ## POST `/knowledge/batch-delete` - 同一知识库内批量删除 按 ID 列表在单个知识库内批量删除知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 属于同一 `kb_id`。 diff --git a/internal/application/service/knowledge_batch_reparse_test.go b/internal/application/service/knowledge_batch_reparse_test.go new file mode 100644 index 00000000..5babdab6 --- /dev/null +++ b/internal/application/service/knowledge_batch_reparse_test.go @@ -0,0 +1,136 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +type reparseFailureKnowledgeRepo struct { + interfaces.KnowledgeRepository + knowledge *types.Knowledge + updateCalls int +} + +func (r *reparseFailureKnowledgeRepo) GetKnowledgeByID( + _ context.Context, + _ uint64, + _ string, +) (*types.Knowledge, error) { + return r.knowledge, nil +} + +func (r *reparseFailureKnowledgeRepo) UpdateKnowledge( + _ context.Context, + _ *types.Knowledge, +) error { + r.updateCalls++ + return nil +} + +func (r *reparseFailureKnowledgeRepo) UpdateKnowledgeColumn( + _ context.Context, + _ string, + _ string, + _ interface{}, +) error { + return nil +} + +type reparseFailureKBService struct { + interfaces.KnowledgeBaseService + kb *types.KnowledgeBase +} + +func (s *reparseFailureKBService) GetKnowledgeBaseByID( + _ context.Context, + _ string, +) (*types.KnowledgeBase, error) { + return s.kb, nil +} + +type failingReparseTaskEnqueuer struct { + err error +} + +func (e failingReparseTaskEnqueuer) Enqueue( + _ *asynq.Task, + _ ...asynq.Option, +) (*asynq.TaskInfo, error) { + return nil, e.err +} + +func TestReparseKnowledgeManualEnqueueFailureIsVisible(t *testing.T) { + enqueueErr := errors.New("queue unavailable") + knowledge := &types.Knowledge{ + ID: "knowledge-1", + TenantID: 7, + KnowledgeBaseID: "kb-1", + Type: types.KnowledgeTypeManual, + ParseStatus: types.ParseStatusCompleted, + EnableStatus: "enabled", + } + require.NoError(t, knowledge.SetManualMetadata( + types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1), + )) + repo := &reparseFailureKnowledgeRepo{knowledge: knowledge} + svc := &knowledgeService{ + repo: repo, + kbService: &reparseFailureKBService{kb: &types.KnowledgeBase{ID: "kb-1"}}, + task: failingReparseTaskEnqueuer{err: enqueueErr}, + } + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + + got, err := svc.ReparseKnowledge(ctx, knowledge.ID, nil) + + require.Error(t, err) + require.Same(t, knowledge, got) + require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus) + require.Equal(t, "disabled", knowledge.EnableStatus) + require.Equal(t, "Failed to enqueue processing task", knowledge.ErrorMessage) + require.GreaterOrEqual(t, repo.updateCalls, 2, "pending and failed states must both be persisted") +} + +func TestRunKnowledgeListReparseSubmissionsReportsPartialFailure(t *testing.T) { + firstErr := errors.New("first failed") + secondErr := errors.New("second failed") + var attempted []string + + outcome, err := runKnowledgeListReparseSubmissions( + []string{"ok-1", "bad-1", "ok-2", "bad-2"}, + func(id string) error { + attempted = append(attempted, id) + switch id { + case "bad-1": + return firstErr + case "bad-2": + return secondErr + default: + return nil + } + }, + ) + + require.Equal(t, []string{"ok-1", "bad-1", "ok-2", "bad-2"}, attempted) + require.Equal(t, knowledgeListReparseOutcome{Submitted: 2, Failed: 2}, outcome) + require.ErrorIs(t, err, asynq.SkipRetry) + require.ErrorIs(t, err, firstErr) + require.ErrorIs(t, err, secondErr) + require.ErrorContains(t, err, "knowledge bad-1") + require.ErrorContains(t, err, "knowledge bad-2") +} + +func TestRunKnowledgeListReparseSubmissionsSucceeds(t *testing.T) { + outcome, err := runKnowledgeListReparseSubmissions( + []string{"knowledge-1", "knowledge-2"}, + func(string) error { return nil }, + ) + + require.NoError(t, err) + require.Equal(t, knowledgeListReparseOutcome{Submitted: 2}, outcome) +} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 6fc9f298..8fbdbb5d 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -2342,9 +2342,8 @@ func (s *knowledgeService) ReparseKnowledge( if _, err := s.enqueueManualProcessing(ctx, existing, meta.Content, true); err != nil { logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err) - existing.ParseStatus = "failed" - existing.ErrorMessage = "Failed to enqueue processing task" - s.repo.UpdateKnowledge(ctx, existing) + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } else { recordReparseStarted() } @@ -2405,7 +2404,8 @@ func (s *knowledgeService) ReparseKnowledge( payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } task := asynq.NewTask( @@ -2416,7 +2416,8 @@ func (s *knowledgeService) ReparseKnowledge( info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID) recordReparseStarted() @@ -2457,7 +2458,8 @@ func (s *knowledgeService) ReparseKnowledge( payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal file URL reparse task payload: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } task := asynq.NewTask( @@ -2468,7 +2470,8 @@ func (s *knowledgeService) ReparseKnowledge( info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } logger.Infof(ctx, "Enqueued file URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID) recordReparseStarted() @@ -2506,7 +2509,8 @@ func (s *knowledgeService) ReparseKnowledge( payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } task := asynq.NewTask( @@ -2517,7 +2521,8 @@ func (s *knowledgeService) ReparseKnowledge( info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err) - return existing, nil + s.markKnowledgeEnqueueFailed(ctx, existing) + return existing, werrors.NewInternalServerError("Failed to submit processing task") } logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID) recordReparseStarted() @@ -2526,7 +2531,13 @@ func (s *knowledgeService) ReparseKnowledge( } logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID) - return existing, nil + existing.ParseStatus = types.ParseStatusFailed + existing.ErrorMessage = "Knowledge has no parseable content" + if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { + logger.Errorf(ctx, "Failed to persist unparseable knowledge state for %s: %v", + secutils.SanitizeForLog(knowledgeID), err) + } + return existing, werrors.NewBadRequestError("Knowledge has no parseable content") } // resetKnowledgeForReparse makes the top-level knowledge state describe the @@ -3745,18 +3756,43 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) - var failed int - for _, id := range payload.KnowledgeIDs { - if _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig); err != nil { - logger.Errorf(ctx, "Failed to reparse knowledge %s: %v", id, err) - failed++ - } - } - - if failed > 0 { - logger.Warnf(ctx, "Knowledge list reparse completed with %d failures out of %d", failed, len(payload.KnowledgeIDs)) - } + outcome, err := runKnowledgeListReparseSubmissions(payload.KnowledgeIDs, func(id string) error { + _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig) + return err + }) logger.Infof(ctx, "Knowledge list reparse task finished: %d submitted, %d failed", - len(payload.KnowledgeIDs)-failed, failed) - return nil + outcome.Submitted, outcome.Failed) + return err +} + +type knowledgeListReparseOutcome struct { + Submitted int + Failed int +} + +// runKnowledgeListReparseSubmissions attempts every item so one bad document +// cannot block the remainder of the batch. A partial failure is non-retryable: +// retrying the wrapper task would destructively reparse items that were already +// submitted successfully. Failed rows remain selectable for an explicit retry. +func runKnowledgeListReparseSubmissions( + ids []string, + submit func(string) error, +) (knowledgeListReparseOutcome, error) { + var outcome knowledgeListReparseOutcome + failures := make([]error, 0) + for _, id := range ids { + if err := submit(id); err != nil { + failures = append(failures, fmt.Errorf("knowledge %s: %w", secutils.SanitizeForLog(id), err)) + outcome.Failed++ + continue + } + outcome.Submitted++ + } + if len(failures) == 0 { + return outcome, nil + } + return outcome, fmt.Errorf( + "%w: batch reparse submitted %d item(s) and failed %d: %w", + asynq.SkipRetry, outcome.Submitted, outcome.Failed, errors.Join(failures...), + ) }