fix(knowledge): leave the root span open while a retry is left (#4357)

* fix(knowledge): leave the root span open while a retry is left

FailSpan closed the attempt root as failed on any main-stage failure, but
asynq retries reuse the payload and record onto the same attempt. Once the
root is terminal FinalizeAttempt returns on its guard, so a retry that
succeeds leaves the root failed while every child span and the document
itself report completed.

Defer the root's terminal status until the runner has no retry left, using
the same counters ProcessDocument already computes isLastRetry from.

* fix(knowledge): distinguish terminal span failures from retryable errors

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
main
The one -Niu 2026-10-17 00:24:34 +08:00 committed by GitHub
parent ad63fc4b8c
commit f5cfa0ff5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 361 additions and 37 deletions

View File

@ -331,6 +331,18 @@ func (s *knowledgeService) endStage(ctx context.Context, kid, name string, outpu
}
func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg string, err error) {
s.failStageWithRetry(ctx, kid, name, code, msg, err, false)
}
// failStageRetryable is for failures returned to the task runner. Remaining
// retry budget alone is insufficient: terminal failures acknowledge the task.
func (s *knowledgeService) failStageRetryable(ctx context.Context, kid, name, code, msg string, err error) {
s.failStageWithRetry(ctx, kid, name, code, msg, err, true)
}
func (s *knowledgeService) failStageWithRetry(
ctx context.Context, kid, name, code, msg string, err error, retryable bool,
) {
a := attemptFromCtx(ctx)
if a <= 0 {
return
@ -339,7 +351,11 @@ func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg s
if span == nil {
return
}
s.tracker().FailSpan(ctx, span, code, msg, err)
if retryable {
s.tracker().FailSpanRetryable(ctx, span, code, msg, err)
} else {
s.tracker().FailSpan(ctx, span, code, msg, err)
}
}
func (s *knowledgeService) skipStage(ctx context.Context, kid, name, reason string) {

View File

@ -274,7 +274,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// enqueue its KB-scoped trigger. Retry only the trigger: appending a
// second pending op would duplicate durable work and its finalizer.
if err := enqueueWikiIngestTrigger(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID); err != nil {
s.tracker().FailSpan(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)
s.tracker().FailSpanRetryable(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)
return fmt.Errorf("retry wiki ingest trigger: %w", err)
}
logger.Infof(ctx, "[KnowledgePostProcess] Re-enqueued wiki ingest trigger for %s", payload.KnowledgeID)
@ -303,7 +303,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
case expectedSubtasks == 0:
completed, err := s.knowledgeRepo.CompleteProcessingWithoutSubtasks(ctx, payload.KnowledgeID)
if err != nil {
s.tracker().FailSpan(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)
s.tracker().FailSpanRetryable(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)
return fmt.Errorf("complete knowledge without enrichment: %w", err)
}
if !completed {
@ -340,7 +340,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// strand the row; retry so the handoff (or dead-letter) happens.
logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",
payload.KnowledgeID, err)
s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)
s.tracker().FailSpanRetryable(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)
return fmt.Errorf("enter finalizing: %w", err)
}
if promoted {

View File

@ -145,9 +145,14 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {
wikiEnqueueFailureKnowledgeRepo: wikiEnqueueFailureKnowledgeRepo{knowledge: knowledge},
completionErr: tc.err, completionSkipped: tc.skipped,
}
tracker, db := setupSpanTrackerTest(t)
ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
_, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
require.NoError(t, err)
queue := &wikiEnqueueFailureTaskQueue{}
svc := &KnowledgePostProcessService{
knowledgeRepo: repo,
spanTracker: tracker,
kbService: &wikiEnqueueFailureKBService{kb: &types.KnowledgeBase{
ID: "kb", TenantID: 7, Type: types.KnowledgeBaseTypeDocument,
AutoTagConfig: &types.AutoTagConfig{Enabled: true},
@ -162,15 +167,24 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {
TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb",
})
require.NoError(t, err)
err = svc.Handle(context.Background(), asynq.NewTask(types.TypeKnowledgePostProcess, payload))
err = svc.Handle(ctx, asynq.NewTask(types.TypeKnowledgePostProcess, payload))
require.ErrorIs(t, err, tc.err)
require.Empty(t, queue.taskTypes)
require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
if tc.err != nil {
status, code, message := rootSpan(t, db, knowledge.ID, attempt)
require.Equal(t, types.SpanStatusRunning, status)
require.Empty(t, code)
require.Empty(t, message)
repo.completionErr = nil
ctx = types.WithTaskRetryMetadata(ctx, 1, 3)
task := asynq.NewTask(types.TypeKnowledgePostProcess, payload)
require.NoError(t, svc.Handle(context.Background(), task))
require.NoError(t, svc.Handle(ctx, task))
require.Equal(t, types.ParseStatusCompleted, knowledge.ParseStatus)
status, code, message = rootSpan(t, db, knowledge.ID, attempt)
require.Equal(t, types.SpanStatusDone, status)
require.Empty(t, code)
require.Empty(t, message)
require.Equal(t, []string{types.TypeKnowledgeAutoTag}, queue.taskTypes)
}
})

View File

@ -769,11 +769,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,
if isLikelyRateLimitError(err) {
code = werrors.ErrCodeEmbeddingRateLimit
}
s.failStage(ctx, knowledge.ID, types.StageEmbedding,
code, "batch index failed", err)
if willRetry {
s.failStageRetryable(ctx, knowledge.ID, types.StageEmbedding,
code, "batch index failed", err)
return fmt.Errorf("batch index: %w", err)
}
s.failStage(ctx, knowledge.ID, types.StageEmbedding,
code, "batch index failed", err)
return nil
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
@ -4241,16 +4243,18 @@ func (s *knowledgeService) convert(
if !isURL {
fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath)
if err != nil {
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to get file", err)
return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to get file", err, taskErr != nil)
return result, taskErr
}
defer fileReader.Close()
contentBytes, err := io.ReadAll(fileReader)
if err != nil {
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to read file", err)
return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to read file", err, taskErr != nil)
return result, taskErr
}
req.FileContent = contentBytes
req.FileName = payload.FileName
@ -4261,9 +4265,10 @@ func (s *knowledgeService) convert(
if err != nil {
code, message := docReaderFailure(err)
logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)
s.failStage(ctx, knowledge.ID, types.StageDocReader,
code, message, err)
return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
code, message, err, taskErr != nil)
return result, taskErr
}
sanitizeReadResult(result)
if result.Error != "" {

View File

@ -7,6 +7,7 @@ import (
"strings"
"testing"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@ -64,6 +65,8 @@ func (s embedFailureModelService) GetEmbeddingModel(
// attempt to keep processing past the missing embedder would panic and fail
// this test, which is exactly the regression worth catching.
func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
modelErr := errors.New("embedding model not found")
knowledge := &types.Knowledge{
ID: "knowledge-1",
@ -75,6 +78,7 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
svc := &knowledgeService{
repo: repo,
modelService: embedFailureModelService{err: modelErr},
spanTracker: tracker,
}
kb := &types.KnowledgeBase{
ID: "kb-1",
@ -83,8 +87,12 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
}
svc.processChunks(context.Background(), kb, knowledge,
_, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
require.NoError(t, err)
ctx = withAttempt(ctx, attempt)
err = svc.processChunks(ctx, kb, knowledge,
[]types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
require.NoError(t, err, "the task is acknowledged even though retry budget remains")
require.Len(t, repo.updates, 1, "the failure must be persisted exactly once")
require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus)
@ -93,6 +101,21 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
"error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage,
)
require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
status, code, message := rootSpan(t, db, knowledge.ID, attempt)
require.Equal(t, types.SpanStatusFailed, status)
require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)
require.Equal(t, knowledge.ErrorMessage, message)
// Housekeeping must not relabel this settled failure as TASK_STALLED.
require.NoError(t, db.Exec(knowledgeTestDDL).Error)
require.NoError(t, db.Exec(
"INSERT INTO knowledges (id, parse_status, error_message) VALUES (?, ?, ?)",
knowledge.ID, knowledge.ParseStatus, knowledge.ErrorMessage).Error)
(&HousekeepingService{db: db}).reapTerminalSpans(ctx)
status, code, message = rootSpan(t, db, knowledge.ID, attempt)
require.Equal(t, types.SpanStatusFailed, status)
require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)
require.Equal(t, knowledge.ErrorMessage, message)
}
// A cancelled context means the run was interrupted, not that the model is

View File

@ -5,6 +5,7 @@ import (
"errors"
"testing"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
@ -48,7 +49,7 @@ func (e *failingIndexEngine) DeleteByKnowledgeIDList(context.Context, []string,
}
func runProcessChunksWithFailingIndex(
ctx context.Context, t *testing.T,
ctx context.Context, t *testing.T, tracker SpanTracker,
) (*indexRetryKnowledgeRepo, *indexRetryChunkRepo, *failingIndexEngine, error) {
t.Helper()
knowledge := &types.Knowledge{
@ -70,6 +71,7 @@ func runProcessChunksWithFailingIndex(
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
svc := &knowledgeService{
repo: repo,
spanTracker: tracker,
chunkRepo: chunks,
modelService: parentChildModelService{embedder: parentChildEmbedder{}},
retrieveEngine: parentChildRetrieveRegistry{engine: engine},
@ -95,9 +97,17 @@ func runProcessChunksWithFailingIndex(
func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {
ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)
tracker, db := setupSpanTrackerTest(t)
_, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")
require.NoError(t, err)
ctx = withAttempt(ctx, attempt)
repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)
require.ErrorContains(t, err, "embedding endpoint unavailable")
status, code, message := rootSpan(t, db, "knowledge-1", attempt)
require.Equal(t, types.SpanStatusRunning, status)
require.Empty(t, code)
require.Empty(t, message)
for _, update := range repo.updates {
require.NotEqual(t, types.ParseStatusFailed, update.ParseStatus)
}
@ -109,9 +119,17 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {
func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {
ctx := types.WithTaskRetryMetadata(context.Background(), 3, 3)
repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)
tracker, db := setupSpanTrackerTest(t)
_, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")
require.NoError(t, err)
ctx = withAttempt(ctx, attempt)
repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)
require.NoError(t, err)
status, code, message := rootSpan(t, db, "knowledge-1", attempt)
require.Equal(t, types.SpanStatusFailed, status)
require.Equal(t, werrors.ErrCodeVectorStoreWriteFailed, code)
require.Equal(t, "batch index failed", message)
require.NotEmpty(t, repo.updates)
last := repo.updates[len(repo.updates)-1]
require.Equal(t, types.ParseStatusFailed, last.ParseStatus)

View File

@ -0,0 +1,113 @@
package service
import (
"context"
"errors"
"io"
"strings"
"testing"
"testing/iotest"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
)
type spanRetryDocReader struct {
interfaces.DocumentReader
result *types.ReadResult
err error
}
func (r *spanRetryDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) {
return r.result, r.err
}
type spanRetryFileService struct {
interfaces.FileService
getErr, readErr error
}
func (s spanRetryFileService) GetFile(context.Context, string) (io.ReadCloser, error) {
if s.getErr != nil {
return nil, s.getErr
}
if s.readErr != nil {
return io.NopCloser(iotest.ErrReader(s.readErr)), nil
}
return io.NopCloser(strings.NewReader("document")), nil
}
// Exercise convert's actual return value: a parser response with Error is
// acknowledged, whereas transport/file errors are returned for queue retry.
func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {
transient := errors.New("temporary read failure")
for _, tc := range []struct {
name string
missingReader bool
parserError string
getErr, readErr, rpcErr error
retryable bool
code string
}{
{name: "reader unavailable", missingReader: true, code: werrors.ErrCodeDocReaderUnavailable},
{
name: "parser rejects document", parserError: "unsupported document",
code: werrors.ErrCodeDocReaderParseFailed,
},
{name: "file fetch fails", getErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
{name: "file read fails", readErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
{name: "reader call fails", rpcErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
} {
for _, retried := range []int{0, 3} {
name := tc.name + "/retry_available"
if retried == 3 {
name = tc.name + "/last_retry"
}
t.Run(name, func(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := types.WithTaskRetryMetadata(context.Background(), retried, 3)
knowledge := &types.Knowledge{
ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "doc.pdf",
ParseStatus: types.ParseStatusProcessing,
}
_, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
require.NoError(t, err)
ctx = withAttempt(ctx, attempt)
svc := &knowledgeService{
repo: &embedFailureKnowledgeRepo{knowledge: knowledge},
spanTracker: tracker,
tenantService: &stubTenantServiceForModelDelete{},
fileSvc: spanRetryFileService{getErr: tc.getErr, readErr: tc.readErr},
}
if !tc.missingReader {
svc.documentReader = &spanRetryDocReader{
result: &types.ReadResult{Error: tc.parserError}, err: tc.rpcErr,
}
}
result, err := svc.convert(ctx, types.DocumentProcessPayload{
FilePath: "doc.pdf", FileName: "doc.pdf", FileType: "pdf",
}, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{}, retried == 3)
require.Nil(t, result)
if tc.retryable {
require.Error(t, err)
} else {
require.NoError(t, err)
}
status, code, message := rootSpan(t, db, knowledge.ID, attempt)
if tc.retryable && retried < 3 {
require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
require.Equal(t, types.SpanStatusRunning, status)
require.Empty(t, code)
require.Empty(t, message)
} else {
require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
require.Equal(t, types.SpanStatusFailed, status)
require.Equal(t, tc.code, code)
require.NotEmpty(t, message)
}
})
}
}
}

View File

@ -36,6 +36,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/gorm"
)
@ -110,9 +111,14 @@ type SpanTracker interface {
// FailSpan marks span as failed and cascade-cancels its
// descendants. errorDetail (a Go error) is recorded verbatim in
// error_detail (truncated to 8 KB) for admin views.
// error_detail (truncated to 8 KB) for admin views. Main pipeline stage
// failures close the root even when unused retry budget remains.
FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)
// FailSpanRetryable records a failure that the caller will return to the
// task runner for retry. It leaves the root open only while retries remain.
FailSpanRetryable(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)
// SkipSpan marks an intentionally not-run span (e.g. multimodal
// on a text-only document). Distinct from cancelled — skipped is
// "we chose not to" while cancelled is "an upstream broke".
@ -493,6 +499,18 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON
}
func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {
t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, false)
}
func (t *spanTracker) FailSpanRetryable(
ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error,
) {
t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, attemptStillHasRetry(ctx))
}
func (t *spanTracker) failSpan(
ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error, willRetry bool,
) {
if span == nil {
return
}
@ -540,13 +558,16 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error
// in StageDependencies (those are siblings, not descendants).
if span.Kind == types.SpanKindStage {
t.cascadeDependentStages(ctx, span, reason)
// Any failure in a MAIN pipeline stage means the attempt is
// done — the parse cannot succeed past this point. Close the
// root span as failed so the UI doesn't show "进行中" forever.
// Optional downstream stages (summary/question/wiki/graph) do
// not poison the attempt: they can fail without invalidating
// the parsed document.
if isMainPipelineStage(span.Name) {
// Only an explicitly retryable failure with remaining budget leaves
// the root open. Retries share the same attempt, so closing it early
// prevents a successful retry from finalizing it as done (#4326).
// Terminal failures must close it even when retry budget is unused:
// callers can record a permanent failure and acknowledge the task.
//
// Optional downstream stages (summary/question/wiki/graph) do not
// poison the attempt: they can fail without invalidating the parsed
// document.
if isMainPipelineStage(span.Name) && !willRetry {
t.FinalizeAttempt(ctx, span.KnowledgeID, span.Attempt,
types.SpanStatusFailed, nil, errorCode, errorMessage)
}
@ -725,12 +746,28 @@ func contains(haystack []string, needle string) bool {
return false
}
// attemptStillHasRetry reports whether retry budget remains, not whether the
// caller will actually request a retry. It reads ProcessDocument's isLastRetry
// counters: the Lite executor's injected metadata wins, otherwise
// the asynq counters. Outside a task runner (a synchronous reparse, a test
// harness) neither is present and the failure is final.
func attemptStillHasRetry(ctx context.Context) bool {
if retried, limit, ok := types.TaskRetryMetadataFromContext(ctx); ok {
return retried < limit
}
retryCount, ok := asynq.GetRetryCount(ctx)
if !ok {
return false
}
maxRetry, ok := asynq.GetMaxRetry(ctx)
return ok && retryCount < maxRetry
}
// isMainPipelineStage reports whether stage is one of the 5 mandatory
// pipeline stages (docreader / chunking / embedding / multimodal /
// postprocess). A failure in any of these terminally invalidates the
// attempt and must close the root as failed. Optional downstream stages
// added later (summary, question, wiki, graph) do NOT match — those
// can fail individually without poisoning the parse result.
// postprocess). A terminal failure in any of these closes the root. Optional
// downstream stages added later (summary, question, wiki, graph) do NOT match
// — those can fail individually without poisoning the parse result.
func isMainPipelineStage(name string) bool {
for _, s := range types.AllStages {
if s == name {
@ -884,10 +921,11 @@ func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string,
func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {
return nil
}
func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}
func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}
func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }
func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}
func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}
func (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {}
func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }
func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span {
return nil
}

View File

@ -537,3 +537,100 @@ func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) {
"kid-reparse").Row().Scan(&newStatus))
assert.Equal(t, types.SpanStatusRunning, newStatus)
}
// rootSpan reads the attempt's root span: the status the UI's error banner and
// its progress bar are both driven by.
func rootSpan(t *testing.T, db *gorm.DB, knowledgeID string, attempt int) (status, code, message string) {
t.Helper()
require.NoError(t, db.Raw(
`SELECT status, COALESCE(error_code, ''), COALESCE(error_message, '')
FROM knowledge_processing_spans
WHERE knowledge_id = ? AND attempt = ? AND kind = 'root'`,
knowledgeID, attempt).Row().Scan(&status, &code, &message))
return status, code, message
}
func TestSpanTracker_FailSpanRetryable_KeepsRootOpenWhileRetriesRemain(t *testing.T) {
ctx := context.Background()
// Every main pipeline stage goes through the same deferral, not just the
// docreader one a report happens to show.
for _, stage := range types.AllStages {
t.Run(stage, func(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
kid := "kid-retry-" + stage
_, attempt, err := tracker.OpenAttempt(ctx, kid, "")
require.NoError(t, err)
// The first run fails in a main stage. asynq retries reuse the
// payload, so the next run records onto this same attempt and may
// still complete it.
first := types.WithTaskRetryMetadata(ctx, 0, 3)
span := tracker.BeginStage(first, kid, attempt, stage, nil)
tracker.FailSpanRetryable(first, span, "STAGE_FAILED", "transient upstream", errors.New("boom"))
status, code, _ := rootSpan(t, db, kid, attempt)
assert.Equal(t, types.SpanStatusRunning, status, "root stays open while a retry is left")
assert.Empty(t, code, "an attempt that can still succeed must not be attributed a failure")
// The retry succeeds. Before this fix the root was already terminal
// here, so FinalizeAttempt returned on its guard and the UI showed
// a red error banner above an all-done trace.
tracker.FinalizeAttempt(types.WithTaskRetryMetadata(ctx, 1, 3),
kid, attempt, types.SpanStatusDone, nil, "", "")
status, code, message := rootSpan(t, db, kid, attempt)
assert.Equal(t, types.SpanStatusDone, status)
assert.Empty(t, code, "no stale error code survives a successful retry")
assert.Empty(t, message)
// A failure arriving after the attempt closed must not reopen it.
late := tracker.BeginStage(first, kid, attempt, stage, nil)
tracker.FailSpanRetryable(types.WithTaskRetryMetadata(ctx, 3, 3),
late, "STAGE_FAILED", "later failure", errors.New("boom"))
status, _, _ = rootSpan(t, db, kid, attempt)
assert.Equal(t, types.SpanStatusDone, status, "a terminal attempt stays terminal")
})
}
}
func TestSpanTracker_FailSpanRetryable_ClosesRootOnTheLastRetry(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid-final", "")
require.NoError(t, err)
last := types.WithTaskRetryMetadata(ctx, 3, 3)
docreader := tracker.BeginStage(last, "kid-final", attempt, types.StageDocReader, nil)
tracker.FailSpanRetryable(last, docreader,
"DOCREADER_PARSE_FAILED", "engine core initialization failed", errors.New("boom"))
status, code, message := rootSpan(t, db, "kid-final", attempt)
assert.Equal(t, types.SpanStatusFailed, status, "exhausted retries must not hang at 进行中")
assert.Equal(t, "DOCREADER_PARSE_FAILED", code)
assert.Equal(t, "engine core initialization failed", message)
// With no task runner in the context at all — a synchronous reparse, or a
// caller outside asynq — the failure is final, as it was before.
_, syncAttempt, err := tracker.OpenAttempt(ctx, "kid-sync", "")
require.NoError(t, err)
post := tracker.BeginStage(ctx, "kid-sync", syncAttempt, types.StagePostProcess, nil)
tracker.FailSpanRetryable(ctx, post, "POSTPROCESS_FAILED", "no retry metadata", errors.New("boom"))
status, _, _ = rootSpan(t, db, "kid-sync", syncAttempt)
assert.Equal(t, types.SpanStatusFailed, status)
}
// Unused queue budget does not make an acknowledged failure retryable.
func TestSpanTracker_FailSpan_ClosesRootWithUnusedRetries(t *testing.T) {
for _, stage := range types.AllStages {
t.Run(stage, func(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
_, attempt, err := tracker.OpenAttempt(ctx, "terminal", "")
require.NoError(t, err)
span := tracker.BeginStage(ctx, "terminal", attempt, stage, nil)
tracker.FailSpan(ctx, span, "PERMANENT_FAILURE", "cannot continue", nil)
status, code, message := rootSpan(t, db, "terminal", attempt)
require.Equal(t, types.SpanStatusFailed, status)
require.Equal(t, "PERMANENT_FAILURE", code)
require.Equal(t, "cannot continue", message)
})
}
}