fix(knowledge): surface batch reparse failures
Signed-off-by: Myang_print <1182470296@qq.com>main
parent
2093fa313a
commit
5780affdfc
|
|
@ -23,6 +23,7 @@
|
|||
| PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 |
|
||||
| PUT | `/knowledge/tags` | 批量更新知识标签 |
|
||||
| GET | `/knowledge/search` | 跨知识库搜索/过滤知识 |
|
||||
| POST | `/knowledge/batch-reparse` | 同一知识库内批量重新解析知识(异步任务) |
|
||||
| POST | `/knowledge/batch-delete` | 同一知识库内批量删除知识(异步任务) |
|
||||
| POST | `/knowledge/move` | 迁移知识到另一知识库(异步任务) |
|
||||
| GET | `/knowledge/move/progress/:task_id` | 查询知识迁移任务进度 |
|
||||
|
|
@ -782,6 +783,50 @@ curl --location --get 'http://localhost:8080/api/v1/knowledge/search' \
|
|||
|
||||
`agent_id=...&` 且该 Agent 的 KB 选择模式为 `none` 时,将直接返回 `data: []` 与 `has_more: false`。
|
||||
|
||||
## POST `/knowledge/batch-reparse` - 同一知识库内批量重新解析
|
||||
|
||||
按 ID 列表在单个知识库内批量重新解析知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 存在并属于同一 `kb_id`。
|
||||
|
||||
**请求体**:
|
||||
|
||||
| 字段 | 类型 | 必填 | 说明 |
|
||||
| ---------------- | -------- | ---- | ------------------------------------------------ |
|
||||
| `kb_id` | string | 是 | 目标知识库 ID |
|
||||
| `ids` | string[] | 是 | 待重新解析的知识 ID 列表(≤ 200) |
|
||||
| `process_config` | object | 否 | 本批次共用的处理配置覆盖;省略时沿用各知识原配置 |
|
||||
|
||||
**请求**:
|
||||
|
||||
```curl
|
||||
curl --location 'http://localhost:8080/api/v1/knowledge/batch-reparse' \
|
||||
--header 'X-API-Key: sk-xxxxx' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"kb_id": "kb-00000001",
|
||||
"ids": [
|
||||
"4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5",
|
||||
"9c8af585-ae15-44ce-8f73-45ad18394651"
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
**响应**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "Batch reparse task submitted",
|
||||
"data": {
|
||||
"task_id": "a1b2c3d4",
|
||||
"reparse_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
HTTP 成功表示批处理包装任务已入队。后台会尝试提交列表中的每个知识;单项提交失败时,该知识会进入 `failed` 状态并保留错误信息,批处理任务也会在运行队列中标记失败。为避免重复清理已成功提交的知识,出现部分失败后不会自动重试整个批次;可筛选失败知识后再次批量重新解析。
|
||||
|
||||
任一 ID 不属于 `kb_id` 或不存在时,返回 400 并整批拒绝。
|
||||
|
||||
## POST `/knowledge/batch-delete` - 同一知识库内批量删除
|
||||
|
||||
按 ID 列表在单个知识库内批量删除知识(异步任务)。单次最多 200 个 ID;服务侧会校验所有 ID 属于同一 `kb_id`。
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type reparseFailureKnowledgeRepo struct {
|
||||
interfaces.KnowledgeRepository
|
||||
knowledge *types.Knowledge
|
||||
updateCalls int
|
||||
}
|
||||
|
||||
func (r *reparseFailureKnowledgeRepo) GetKnowledgeByID(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ string,
|
||||
) (*types.Knowledge, error) {
|
||||
return r.knowledge, nil
|
||||
}
|
||||
|
||||
func (r *reparseFailureKnowledgeRepo) UpdateKnowledge(
|
||||
_ context.Context,
|
||||
_ *types.Knowledge,
|
||||
) error {
|
||||
r.updateCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *reparseFailureKnowledgeRepo) UpdateKnowledgeColumn(
|
||||
_ context.Context,
|
||||
_ string,
|
||||
_ string,
|
||||
_ interface{},
|
||||
) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type reparseFailureKBService struct {
|
||||
interfaces.KnowledgeBaseService
|
||||
kb *types.KnowledgeBase
|
||||
}
|
||||
|
||||
func (s *reparseFailureKBService) GetKnowledgeBaseByID(
|
||||
_ context.Context,
|
||||
_ string,
|
||||
) (*types.KnowledgeBase, error) {
|
||||
return s.kb, nil
|
||||
}
|
||||
|
||||
type failingReparseTaskEnqueuer struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e failingReparseTaskEnqueuer) Enqueue(
|
||||
_ *asynq.Task,
|
||||
_ ...asynq.Option,
|
||||
) (*asynq.TaskInfo, error) {
|
||||
return nil, e.err
|
||||
}
|
||||
|
||||
func TestReparseKnowledgeManualEnqueueFailureIsVisible(t *testing.T) {
|
||||
enqueueErr := errors.New("queue unavailable")
|
||||
knowledge := &types.Knowledge{
|
||||
ID: "knowledge-1",
|
||||
TenantID: 7,
|
||||
KnowledgeBaseID: "kb-1",
|
||||
Type: types.KnowledgeTypeManual,
|
||||
ParseStatus: types.ParseStatusCompleted,
|
||||
EnableStatus: "enabled",
|
||||
}
|
||||
require.NoError(t, knowledge.SetManualMetadata(
|
||||
types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1),
|
||||
))
|
||||
repo := &reparseFailureKnowledgeRepo{knowledge: knowledge}
|
||||
svc := &knowledgeService{
|
||||
repo: repo,
|
||||
kbService: &reparseFailureKBService{kb: &types.KnowledgeBase{ID: "kb-1"}},
|
||||
task: failingReparseTaskEnqueuer{err: enqueueErr},
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
|
||||
|
||||
got, err := svc.ReparseKnowledge(ctx, knowledge.ID, nil)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Same(t, knowledge, got)
|
||||
require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
|
||||
require.Equal(t, "disabled", knowledge.EnableStatus)
|
||||
require.Equal(t, "Failed to enqueue processing task", knowledge.ErrorMessage)
|
||||
require.GreaterOrEqual(t, repo.updateCalls, 2, "pending and failed states must both be persisted")
|
||||
}
|
||||
|
||||
func TestRunKnowledgeListReparseSubmissionsReportsPartialFailure(t *testing.T) {
|
||||
firstErr := errors.New("first failed")
|
||||
secondErr := errors.New("second failed")
|
||||
var attempted []string
|
||||
|
||||
outcome, err := runKnowledgeListReparseSubmissions(
|
||||
[]string{"ok-1", "bad-1", "ok-2", "bad-2"},
|
||||
func(id string) error {
|
||||
attempted = append(attempted, id)
|
||||
switch id {
|
||||
case "bad-1":
|
||||
return firstErr
|
||||
case "bad-2":
|
||||
return secondErr
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
require.Equal(t, []string{"ok-1", "bad-1", "ok-2", "bad-2"}, attempted)
|
||||
require.Equal(t, knowledgeListReparseOutcome{Submitted: 2, Failed: 2}, outcome)
|
||||
require.ErrorIs(t, err, asynq.SkipRetry)
|
||||
require.ErrorIs(t, err, firstErr)
|
||||
require.ErrorIs(t, err, secondErr)
|
||||
require.ErrorContains(t, err, "knowledge bad-1")
|
||||
require.ErrorContains(t, err, "knowledge bad-2")
|
||||
}
|
||||
|
||||
func TestRunKnowledgeListReparseSubmissionsSucceeds(t *testing.T) {
|
||||
outcome, err := runKnowledgeListReparseSubmissions(
|
||||
[]string{"knowledge-1", "knowledge-2"},
|
||||
func(string) error { return nil },
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, knowledgeListReparseOutcome{Submitted: 2}, outcome)
|
||||
}
|
||||
|
|
@ -2342,9 +2342,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
|
||||
if _, err := s.enqueueManualProcessing(ctx, existing, meta.Content, true); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err)
|
||||
existing.ParseStatus = "failed"
|
||||
existing.ErrorMessage = "Failed to enqueue processing task"
|
||||
s.repo.UpdateKnowledge(ctx, existing)
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
} else {
|
||||
recordReparseStarted()
|
||||
}
|
||||
|
|
@ -2405,7 +2404,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
|
||||
task := asynq.NewTask(
|
||||
|
|
@ -2416,7 +2416,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
|
||||
recordReparseStarted()
|
||||
|
|
@ -2457,7 +2458,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal file URL reparse task payload: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
|
||||
task := asynq.NewTask(
|
||||
|
|
@ -2468,7 +2470,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued file URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
|
||||
recordReparseStarted()
|
||||
|
|
@ -2506,7 +2509,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
|
||||
task := asynq.NewTask(
|
||||
|
|
@ -2517,7 +2521,8 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
|
||||
return existing, nil
|
||||
s.markKnowledgeEnqueueFailed(ctx, existing)
|
||||
return existing, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
|
||||
recordReparseStarted()
|
||||
|
|
@ -2526,7 +2531,13 @@ func (s *knowledgeService) ReparseKnowledge(
|
|||
}
|
||||
|
||||
logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID)
|
||||
return existing, nil
|
||||
existing.ParseStatus = types.ParseStatusFailed
|
||||
existing.ErrorMessage = "Knowledge has no parseable content"
|
||||
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
|
||||
logger.Errorf(ctx, "Failed to persist unparseable knowledge state for %s: %v",
|
||||
secutils.SanitizeForLog(knowledgeID), err)
|
||||
}
|
||||
return existing, werrors.NewBadRequestError("Knowledge has no parseable content")
|
||||
}
|
||||
|
||||
// resetKnowledgeForReparse makes the top-level knowledge state describe the
|
||||
|
|
@ -3745,18 +3756,43 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a
|
|||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
||||
|
||||
var failed int
|
||||
for _, id := range payload.KnowledgeIDs {
|
||||
if _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig); err != nil {
|
||||
logger.Errorf(ctx, "Failed to reparse knowledge %s: %v", id, err)
|
||||
failed++
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
logger.Warnf(ctx, "Knowledge list reparse completed with %d failures out of %d", failed, len(payload.KnowledgeIDs))
|
||||
}
|
||||
outcome, err := runKnowledgeListReparseSubmissions(payload.KnowledgeIDs, func(id string) error {
|
||||
_, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig)
|
||||
return err
|
||||
})
|
||||
logger.Infof(ctx, "Knowledge list reparse task finished: %d submitted, %d failed",
|
||||
len(payload.KnowledgeIDs)-failed, failed)
|
||||
return nil
|
||||
outcome.Submitted, outcome.Failed)
|
||||
return err
|
||||
}
|
||||
|
||||
type knowledgeListReparseOutcome struct {
|
||||
Submitted int
|
||||
Failed int
|
||||
}
|
||||
|
||||
// runKnowledgeListReparseSubmissions attempts every item so one bad document
|
||||
// cannot block the remainder of the batch. A partial failure is non-retryable:
|
||||
// retrying the wrapper task would destructively reparse items that were already
|
||||
// submitted successfully. Failed rows remain selectable for an explicit retry.
|
||||
func runKnowledgeListReparseSubmissions(
|
||||
ids []string,
|
||||
submit func(string) error,
|
||||
) (knowledgeListReparseOutcome, error) {
|
||||
var outcome knowledgeListReparseOutcome
|
||||
failures := make([]error, 0)
|
||||
for _, id := range ids {
|
||||
if err := submit(id); err != nil {
|
||||
failures = append(failures, fmt.Errorf("knowledge %s: %w", secutils.SanitizeForLog(id), err))
|
||||
outcome.Failed++
|
||||
continue
|
||||
}
|
||||
outcome.Submitted++
|
||||
}
|
||||
if len(failures) == 0 {
|
||||
return outcome, nil
|
||||
}
|
||||
return outcome, fmt.Errorf(
|
||||
"%w: batch reparse submitted %d item(s) and failed %d: %w",
|
||||
asynq.SkipRetry, outcome.Submitted, outcome.Failed, errors.Join(failures...),
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue