fix(knowledge): surface batch reparse failures

Signed-off-by: Myang_print <1182470296@qq.com>
main
Myang_print 2026-07-31 12:55:50 +08:00 committed by lyingbug
parent 2093fa313a
commit 5780affdfc
3 changed files with 240 additions and 23 deletions

View File

@ -23,6 +23,7 @@
| PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 |
| PUT | `/knowledge/tags` | 批量更新知识标签 |
| GET | `/knowledge/search` | 跨知识库搜索/过滤知识 |
| POST | `/knowledge/batch-reparse` | 同一知识库内批量重新解析知识(异步任务) |
| POST | `/knowledge/batch-delete` | 同一知识库内批量删除知识(异步任务) |
| POST | `/knowledge/move` | 迁移知识到另一知识库(异步任务) |
| GET | `/knowledge/move/progress/:task_id` | 查询知识迁移任务进度 |
@ -782,6 +783,50 @@ curl --location --get 'http://localhost:8080/api/v1/knowledge/search' \
`agent_id=...&` 且该 Agent 的 KB 选择模式为 `none` 时,将直接返回 `data: []` 与 `has_more: false`。
## POST `/knowledge/batch-reparse` - 同一知识库内批量重新解析
按 ID 列表在单个知识库内批量重新解析知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 存在并属于同一 `kb_id`。
**请求体**:
| 字段 | 类型 | 必填 | 说明 |
| ---------------- | -------- | ---- | ------------------------------------------------ |
| `kb_id` | string | 是 | 目标知识库 ID |
| `ids` | string[] | 是 | 待重新解析的知识 ID 列表(≤ 200) |
| `process_config` | object | 否 | 本批次共用的处理配置覆盖;省略时沿用各知识原配置 |
**请求**:
```curl
curl --location 'http://localhost:8080/api/v1/knowledge/batch-reparse' \
--header 'X-API-Key: sk-xxxxx' \
--header 'Content-Type: application/json' \
--data '{
"kb_id": "kb-00000001",
"ids": [
"4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5",
"9c8af585-ae15-44ce-8f73-45ad18394651"
]
}'
```
**响应**:
```json
{
"success": true,
"message": "Batch reparse task submitted",
"data": {
"task_id": "a1b2c3d4",
"reparse_count": 2
}
}
```
HTTP 成功表示批处理包装任务已入队。后台会尝试提交列表中的每个知识;单项提交失败时,该知识会进入 `failed` 状态并保留错误信息,批处理任务也会在运行队列中标记失败。为避免重复清理已成功提交的知识,出现部分失败后不会自动重试整个批次;可筛选失败知识后再次批量重新解析。
任一 ID 不属于 `kb_id` 或不存在时,返回 400 并整批拒绝。
## POST `/knowledge/batch-delete` - 同一知识库内批量删除
按 ID 列表在单个知识库内批量删除知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 属于同一 `kb_id`。

View File

@ -0,0 +1,136 @@
package service
import (
"context"
"errors"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/require"
)
type reparseFailureKnowledgeRepo struct {
interfaces.KnowledgeRepository
knowledge *types.Knowledge
updateCalls int
}
func (r *reparseFailureKnowledgeRepo) GetKnowledgeByID(
_ context.Context,
_ uint64,
_ string,
) (*types.Knowledge, error) {
return r.knowledge, nil
}
func (r *reparseFailureKnowledgeRepo) UpdateKnowledge(
_ context.Context,
_ *types.Knowledge,
) error {
r.updateCalls++
return nil
}
func (r *reparseFailureKnowledgeRepo) UpdateKnowledgeColumn(
_ context.Context,
_ string,
_ string,
_ interface{},
) error {
return nil
}
type reparseFailureKBService struct {
interfaces.KnowledgeBaseService
kb *types.KnowledgeBase
}
func (s *reparseFailureKBService) GetKnowledgeBaseByID(
_ context.Context,
_ string,
) (*types.KnowledgeBase, error) {
return s.kb, nil
}
type failingReparseTaskEnqueuer struct {
err error
}
func (e failingReparseTaskEnqueuer) Enqueue(
_ *asynq.Task,
_ ...asynq.Option,
) (*asynq.TaskInfo, error) {
return nil, e.err
}
func TestReparseKnowledgeManualEnqueueFailureIsVisible(t *testing.T) {
enqueueErr := errors.New("queue unavailable")
knowledge := &types.Knowledge{
ID: "knowledge-1",
TenantID: 7,
KnowledgeBaseID: "kb-1",
Type: types.KnowledgeTypeManual,
ParseStatus: types.ParseStatusCompleted,
EnableStatus: "enabled",
}
require.NoError(t, knowledge.SetManualMetadata(
types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1),
))
repo := &reparseFailureKnowledgeRepo{knowledge: knowledge}
svc := &knowledgeService{
repo: repo,
kbService: &reparseFailureKBService{kb: &types.KnowledgeBase{ID: "kb-1"}},
task: failingReparseTaskEnqueuer{err: enqueueErr},
}
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
got, err := svc.ReparseKnowledge(ctx, knowledge.ID, nil)
require.Error(t, err)
require.Same(t, knowledge, got)
require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
require.Equal(t, "disabled", knowledge.EnableStatus)
require.Equal(t, "Failed to enqueue processing task", knowledge.ErrorMessage)
require.GreaterOrEqual(t, repo.updateCalls, 2, "pending and failed states must both be persisted")
}
func TestRunKnowledgeListReparseSubmissionsReportsPartialFailure(t *testing.T) {
firstErr := errors.New("first failed")
secondErr := errors.New("second failed")
var attempted []string
outcome, err := runKnowledgeListReparseSubmissions(
[]string{"ok-1", "bad-1", "ok-2", "bad-2"},
func(id string) error {
attempted = append(attempted, id)
switch id {
case "bad-1":
return firstErr
case "bad-2":
return secondErr
default:
return nil
}
},
)
require.Equal(t, []string{"ok-1", "bad-1", "ok-2", "bad-2"}, attempted)
require.Equal(t, knowledgeListReparseOutcome{Submitted: 2, Failed: 2}, outcome)
require.ErrorIs(t, err, asynq.SkipRetry)
require.ErrorIs(t, err, firstErr)
require.ErrorIs(t, err, secondErr)
require.ErrorContains(t, err, "knowledge bad-1")
require.ErrorContains(t, err, "knowledge bad-2")
}
func TestRunKnowledgeListReparseSubmissionsSucceeds(t *testing.T) {
outcome, err := runKnowledgeListReparseSubmissions(
[]string{"knowledge-1", "knowledge-2"},
func(string) error { return nil },
)
require.NoError(t, err)
require.Equal(t, knowledgeListReparseOutcome{Submitted: 2}, outcome)
}

View File

@ -2342,9 +2342,8 @@ func (s *knowledgeService) ReparseKnowledge(
if _, err := s.enqueueManualProcessing(ctx, existing, meta.Content, true); err != nil {
logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err)
existing.ParseStatus = "failed"
existing.ErrorMessage = "Failed to enqueue processing task"
s.repo.UpdateKnowledge(ctx, existing)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
} else {
recordReparseStarted()
}
@ -2405,7 +2404,8 @@ func (s *knowledgeService) ReparseKnowledge(
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
@ -2416,7 +2416,8 @@ func (s *knowledgeService) ReparseKnowledge(
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
@ -2457,7 +2458,8 @@ func (s *knowledgeService) ReparseKnowledge(
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal file URL reparse task payload: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
@ -2468,7 +2470,8 @@ func (s *knowledgeService) ReparseKnowledge(
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued file URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
@ -2506,7 +2509,8 @@ func (s *knowledgeService) ReparseKnowledge(
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
@ -2517,7 +2521,8 @@ func (s *knowledgeService) ReparseKnowledge(
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
return existing, nil
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
@ -2526,7 +2531,13 @@ func (s *knowledgeService) ReparseKnowledge(
}
logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID)
return existing, nil
existing.ParseStatus = types.ParseStatusFailed
existing.ErrorMessage = "Knowledge has no parseable content"
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to persist unparseable knowledge state for %s: %v",
secutils.SanitizeForLog(knowledgeID), err)
}
return existing, werrors.NewBadRequestError("Knowledge has no parseable content")
}
// resetKnowledgeForReparse makes the top-level knowledge state describe the
@ -3745,18 +3756,43 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
var failed int
for _, id := range payload.KnowledgeIDs {
if _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig); err != nil {
logger.Errorf(ctx, "Failed to reparse knowledge %s: %v", id, err)
failed++
}
}
if failed > 0 {
logger.Warnf(ctx, "Knowledge list reparse completed with %d failures out of %d", failed, len(payload.KnowledgeIDs))
}
outcome, err := runKnowledgeListReparseSubmissions(payload.KnowledgeIDs, func(id string) error {
_, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig)
return err
})
logger.Infof(ctx, "Knowledge list reparse task finished: %d submitted, %d failed",
len(payload.KnowledgeIDs)-failed, failed)
return nil
outcome.Submitted, outcome.Failed)
return err
}
type knowledgeListReparseOutcome struct {
Submitted int
Failed int
}
// runKnowledgeListReparseSubmissions attempts every item so one bad document
// cannot block the remainder of the batch. A partial failure is non-retryable:
// retrying the wrapper task would destructively reparse items that were already
// submitted successfully. Failed rows remain selectable for an explicit retry.
func runKnowledgeListReparseSubmissions(
ids []string,
submit func(string) error,
) (knowledgeListReparseOutcome, error) {
var outcome knowledgeListReparseOutcome
failures := make([]error, 0)
for _, id := range ids {
if err := submit(id); err != nil {
failures = append(failures, fmt.Errorf("knowledge %s: %w", secutils.SanitizeForLog(id), err))
outcome.Failed++
continue
}
outcome.Submitted++
}
if len(failures) == 0 {
return outcome, nil
}
return outcome, fmt.Errorf(
"%w: batch reparse submitted %d item(s) and failed %d: %w",
asynq.SkipRetry, outcome.Submitted, outcome.Failed, errors.Join(failures...),
)
}