diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index c7126a27..7a337673 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -331,6 +331,18 @@ func (s *knowledgeService) endStage(ctx context.Context, kid, name string, outpu } func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg string, err error) { + s.failStageWithRetry(ctx, kid, name, code, msg, err, false) +} + +// failStageRetryable is for failures returned to the task runner. Remaining +// retry budget alone is insufficient: terminal failures acknowledge the task. +func (s *knowledgeService) failStageRetryable(ctx context.Context, kid, name, code, msg string, err error) { + s.failStageWithRetry(ctx, kid, name, code, msg, err, true) +} + +func (s *knowledgeService) failStageWithRetry( + ctx context.Context, kid, name, code, msg string, err error, retryable bool, +) { a := attemptFromCtx(ctx) if a <= 0 { return @@ -339,7 +351,11 @@ func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg s if span == nil { return } - s.tracker().FailSpan(ctx, span, code, msg, err) + if retryable { + s.tracker().FailSpanRetryable(ctx, span, code, msg, err) + } else { + s.tracker().FailSpan(ctx, span, code, msg, err) + } } func (s *knowledgeService) skipStage(ctx context.Context, kid, name, reason string) { diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index 67b9f90b..1f889e50 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -274,7 +274,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // enqueue its KB-scoped trigger. Retry only the trigger: appending a // second pending op would duplicate durable work and its finalizer. if err := enqueueWikiIngestTrigger(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID); err != nil { - s.tracker().FailSpan(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err) + s.tracker().FailSpanRetryable(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err) return fmt.Errorf("retry wiki ingest trigger: %w", err) } logger.Infof(ctx, "[KnowledgePostProcess] Re-enqueued wiki ingest trigger for %s", payload.KnowledgeID) @@ -303,7 +303,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta case expectedSubtasks == 0: completed, err := s.knowledgeRepo.CompleteProcessingWithoutSubtasks(ctx, payload.KnowledgeID) if err != nil { - s.tracker().FailSpan(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err) + s.tracker().FailSpanRetryable(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err) return fmt.Errorf("complete knowledge without enrichment: %w", err) } if !completed { @@ -340,7 +340,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // strand the row; retry so the handoff (or dead-letter) happens. logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v", payload.KnowledgeID, err) - s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err) + s.tracker().FailSpanRetryable(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err) return fmt.Errorf("enter finalizing: %w", err) } if promoted { diff --git a/internal/application/service/knowledge_post_process_summary_test.go b/internal/application/service/knowledge_post_process_summary_test.go index 1c1e65df..5e4e9f76 100644 --- a/internal/application/service/knowledge_post_process_summary_test.go +++ b/internal/application/service/knowledge_post_process_summary_test.go @@ -145,9 +145,14 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) { wikiEnqueueFailureKnowledgeRepo: wikiEnqueueFailureKnowledgeRepo{knowledge: knowledge}, completionErr: tc.err, completionSkipped: tc.skipped, } + tracker, db := setupSpanTrackerTest(t) + ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3) + _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "") + require.NoError(t, err) queue := &wikiEnqueueFailureTaskQueue{} svc := &KnowledgePostProcessService{ knowledgeRepo: repo, + spanTracker: tracker, kbService: &wikiEnqueueFailureKBService{kb: &types.KnowledgeBase{ ID: "kb", TenantID: 7, Type: types.KnowledgeBaseTypeDocument, AutoTagConfig: &types.AutoTagConfig{Enabled: true}, @@ -162,15 +167,24 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) { TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", }) require.NoError(t, err) - err = svc.Handle(context.Background(), asynq.NewTask(types.TypeKnowledgePostProcess, payload)) + err = svc.Handle(ctx, asynq.NewTask(types.TypeKnowledgePostProcess, payload)) require.ErrorIs(t, err, tc.err) require.Empty(t, queue.taskTypes) require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus) if tc.err != nil { + status, code, message := rootSpan(t, db, knowledge.ID, attempt) + require.Equal(t, types.SpanStatusRunning, status) + require.Empty(t, code) + require.Empty(t, message) repo.completionErr = nil + ctx = types.WithTaskRetryMetadata(ctx, 1, 3) task := asynq.NewTask(types.TypeKnowledgePostProcess, payload) - require.NoError(t, svc.Handle(context.Background(), task)) + require.NoError(t, svc.Handle(ctx, task)) require.Equal(t, types.ParseStatusCompleted, knowledge.ParseStatus) + status, code, message = rootSpan(t, db, knowledge.ID, attempt) + require.Equal(t, types.SpanStatusDone, status) + require.Empty(t, code) + require.Empty(t, message) require.Equal(t, []string{types.TypeKnowledgeAutoTag}, queue.taskTypes) } }) diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 96e51b57..7ffd22f0 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -769,11 +769,13 @@ func (s *knowledgeService) processChunks(ctx context.Context, if isLikelyRateLimitError(err) { code = werrors.ErrCodeEmbeddingRateLimit } - s.failStage(ctx, knowledge.ID, types.StageEmbedding, - code, "batch index failed", err) if willRetry { + s.failStageRetryable(ctx, knowledge.ID, types.StageEmbedding, + code, "batch index failed", err) return fmt.Errorf("batch index: %w", err) } + s.failStage(ctx, knowledge.ID, types.StageEmbedding, + code, "batch index failed", err) return nil } logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList)) @@ -4241,16 +4243,18 @@ func (s *knowledgeService) convert( if !isURL { fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath) if err != nil { - s.failStage(ctx, knowledge.ID, types.StageDocReader, - werrors.ErrCodeDocReaderParseFailed, "failed to get file", err) - return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err) + result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err) + s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader, + werrors.ErrCodeDocReaderParseFailed, "failed to get file", err, taskErr != nil) + return result, taskErr } defer fileReader.Close() contentBytes, err := io.ReadAll(fileReader) if err != nil { - s.failStage(ctx, knowledge.ID, types.StageDocReader, - werrors.ErrCodeDocReaderParseFailed, "failed to read file", err) - return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err) + result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err) + s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader, + werrors.ErrCodeDocReaderParseFailed, "failed to read file", err, taskErr != nil) + return result, taskErr } req.FileContent = contentBytes req.FileName = payload.FileName @@ -4261,9 +4265,10 @@ func (s *knowledgeService) convert( if err != nil { code, message := docReaderFailure(err) logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err) - s.failStage(ctx, knowledge.ID, types.StageDocReader, - code, message, err) - return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message) + result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message) + s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader, + code, message, err, taskErr != nil) + return result, taskErr } sanitizeReadResult(result) if result.Error != "" { diff --git a/internal/application/service/knowledge_process_embedding_failure_test.go b/internal/application/service/knowledge_process_embedding_failure_test.go index 238ed02f..18842baf 100644 --- a/internal/application/service/knowledge_process_embedding_failure_test.go +++ b/internal/application/service/knowledge_process_embedding_failure_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -64,6 +65,8 @@ func (s embedFailureModelService) GetEmbeddingModel( // attempt to keep processing past the missing embedder would panic and fail // this test, which is exactly the regression worth catching. func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3) modelErr := errors.New("embedding model not found") knowledge := &types.Knowledge{ ID: "knowledge-1", @@ -75,6 +78,7 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) svc := &knowledgeService{ repo: repo, modelService: embedFailureModelService{err: modelErr}, + spanTracker: tracker, } kb := &types.KnowledgeBase{ ID: "kb-1", @@ -83,8 +87,12 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) IndexingStrategy: types.IndexingStrategy{VectorEnabled: true}, } - svc.processChunks(context.Background(), kb, knowledge, + _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "") + require.NoError(t, err) + ctx = withAttempt(ctx, attempt) + err = svc.processChunks(ctx, kb, knowledge, []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}}) + require.NoError(t, err, "the task is acknowledged even though retry budget remains") require.Len(t, repo.updates, 1, "the failure must be persisted exactly once") require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus) @@ -93,6 +101,21 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) "error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage, ) require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus) + status, code, message := rootSpan(t, db, knowledge.ID, attempt) + require.Equal(t, types.SpanStatusFailed, status) + require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code) + require.Equal(t, knowledge.ErrorMessage, message) + + // Housekeeping must not relabel this settled failure as TASK_STALLED. + require.NoError(t, db.Exec(knowledgeTestDDL).Error) + require.NoError(t, db.Exec( + "INSERT INTO knowledges (id, parse_status, error_message) VALUES (?, ?, ?)", + knowledge.ID, knowledge.ParseStatus, knowledge.ErrorMessage).Error) + (&HousekeepingService{db: db}).reapTerminalSpans(ctx) + status, code, message = rootSpan(t, db, knowledge.ID, attempt) + require.Equal(t, types.SpanStatusFailed, status) + require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code) + require.Equal(t, knowledge.ErrorMessage, message) } // A cancelled context means the run was interrupted, not that the model is diff --git a/internal/application/service/knowledge_process_index_retry_test.go b/internal/application/service/knowledge_process_index_retry_test.go index 89bcf737..e6703a72 100644 --- a/internal/application/service/knowledge_process_index_retry_test.go +++ b/internal/application/service/knowledge_process_index_retry_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/types" "github.com/stretchr/testify/require" @@ -48,7 +49,7 @@ func (e *failingIndexEngine) DeleteByKnowledgeIDList(context.Context, []string, } func runProcessChunksWithFailingIndex( - ctx context.Context, t *testing.T, + ctx context.Context, t *testing.T, tracker SpanTracker, ) (*indexRetryKnowledgeRepo, *indexRetryChunkRepo, *failingIndexEngine, error) { t.Helper() knowledge := &types.Knowledge{ @@ -70,6 +71,7 @@ func runProcessChunksWithFailingIndex( ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) svc := &knowledgeService{ repo: repo, + spanTracker: tracker, chunkRepo: chunks, modelService: parentChildModelService{embedder: parentChildEmbedder{}}, retrieveEngine: parentChildRetrieveRegistry{engine: engine}, @@ -95,9 +97,17 @@ func runProcessChunksWithFailingIndex( func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) { ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3) - repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t) + tracker, db := setupSpanTrackerTest(t) + _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "") + require.NoError(t, err) + ctx = withAttempt(ctx, attempt) + repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker) require.ErrorContains(t, err, "embedding endpoint unavailable") + status, code, message := rootSpan(t, db, "knowledge-1", attempt) + require.Equal(t, types.SpanStatusRunning, status) + require.Empty(t, code) + require.Empty(t, message) for _, update := range repo.updates { require.NotEqual(t, types.ParseStatusFailed, update.ParseStatus) } @@ -109,9 +119,17 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) { func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) { ctx := types.WithTaskRetryMetadata(context.Background(), 3, 3) - repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t) + tracker, db := setupSpanTrackerTest(t) + _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "") + require.NoError(t, err) + ctx = withAttempt(ctx, attempt) + repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker) require.NoError(t, err) + status, code, message := rootSpan(t, db, "knowledge-1", attempt) + require.Equal(t, types.SpanStatusFailed, status) + require.Equal(t, werrors.ErrCodeVectorStoreWriteFailed, code) + require.Equal(t, "batch index failed", message) require.NotEmpty(t, repo.updates) last := repo.updates[len(repo.updates)-1] require.Equal(t, types.ParseStatusFailed, last.ParseStatus) diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.go new file mode 100644 index 00000000..2eca9adc --- /dev/null +++ b/internal/application/service/knowledge_process_span_retry_test.go @@ -0,0 +1,113 @@ +package service + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "testing/iotest" + + werrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +type spanRetryDocReader struct { + interfaces.DocumentReader + result *types.ReadResult + err error +} + +func (r *spanRetryDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) { + return r.result, r.err +} + +type spanRetryFileService struct { + interfaces.FileService + getErr, readErr error +} + +func (s spanRetryFileService) GetFile(context.Context, string) (io.ReadCloser, error) { + if s.getErr != nil { + return nil, s.getErr + } + if s.readErr != nil { + return io.NopCloser(iotest.ErrReader(s.readErr)), nil + } + return io.NopCloser(strings.NewReader("document")), nil +} + +// Exercise convert's actual return value: a parser response with Error is +// acknowledged, whereas transport/file errors are returned for queue retry. +func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) { + transient := errors.New("temporary read failure") + for _, tc := range []struct { + name string + missingReader bool + parserError string + getErr, readErr, rpcErr error + retryable bool + code string + }{ + {name: "reader unavailable", missingReader: true, code: werrors.ErrCodeDocReaderUnavailable}, + { + name: "parser rejects document", parserError: "unsupported document", + code: werrors.ErrCodeDocReaderParseFailed, + }, + {name: "file fetch fails", getErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed}, + {name: "file read fails", readErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed}, + {name: "reader call fails", rpcErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed}, + } { + for _, retried := range []int{0, 3} { + name := tc.name + "/retry_available" + if retried == 3 { + name = tc.name + "/last_retry" + } + t.Run(name, func(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := types.WithTaskRetryMetadata(context.Background(), retried, 3) + knowledge := &types.Knowledge{ + ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "doc.pdf", + ParseStatus: types.ParseStatusProcessing, + } + _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "") + require.NoError(t, err) + ctx = withAttempt(ctx, attempt) + svc := &knowledgeService{ + repo: &embedFailureKnowledgeRepo{knowledge: knowledge}, + spanTracker: tracker, + tenantService: &stubTenantServiceForModelDelete{}, + fileSvc: spanRetryFileService{getErr: tc.getErr, readErr: tc.readErr}, + } + if !tc.missingReader { + svc.documentReader = &spanRetryDocReader{ + result: &types.ReadResult{Error: tc.parserError}, err: tc.rpcErr, + } + } + result, err := svc.convert(ctx, types.DocumentProcessPayload{ + FilePath: "doc.pdf", FileName: "doc.pdf", FileType: "pdf", + }, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{}, retried == 3) + require.Nil(t, result) + if tc.retryable { + require.Error(t, err) + } else { + require.NoError(t, err) + } + status, code, message := rootSpan(t, db, knowledge.ID, attempt) + if tc.retryable && retried < 3 { + require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus) + require.Equal(t, types.SpanStatusRunning, status) + require.Empty(t, code) + require.Empty(t, message) + } else { + require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus) + require.Equal(t, types.SpanStatusFailed, status) + require.Equal(t, tc.code, code) + require.NotEmpty(t, message) + } + }) + } + } +} diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go index 100c27d0..f5bac889 100644 --- a/internal/application/service/knowledge_span_tracker.go +++ b/internal/application/service/knowledge_span_tracker.go @@ -36,6 +36,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" + "github.com/hibiken/asynq" "gorm.io/gorm" ) @@ -110,9 +111,14 @@ type SpanTracker interface { // FailSpan marks span as failed and cascade-cancels its // descendants. errorDetail (a Go error) is recorded verbatim in - // error_detail (truncated to 8 KB) for admin views. + // error_detail (truncated to 8 KB) for admin views. Main pipeline stage + // failures close the root even when unused retry budget remains. FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) + // FailSpanRetryable records a failure that the caller will return to the + // task runner for retry. It leaves the root open only while retries remain. + FailSpanRetryable(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) + // SkipSpan marks an intentionally not-run span (e.g. multimodal // on a text-only document). Distinct from cancelled — skipped is // "we chose not to" while cancelled is "an upstream broke". @@ -493,6 +499,18 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON } func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) { + t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, false) +} + +func (t *spanTracker) FailSpanRetryable( + ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error, +) { + t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, attemptStillHasRetry(ctx)) +} + +func (t *spanTracker) failSpan( + ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error, willRetry bool, +) { if span == nil { return } @@ -540,13 +558,16 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error // in StageDependencies (those are siblings, not descendants). if span.Kind == types.SpanKindStage { t.cascadeDependentStages(ctx, span, reason) - // Any failure in a MAIN pipeline stage means the attempt is - // done — the parse cannot succeed past this point. Close the - // root span as failed so the UI doesn't show "进行中" forever. - // Optional downstream stages (summary/question/wiki/graph) do - // not poison the attempt: they can fail without invalidating - // the parsed document. - if isMainPipelineStage(span.Name) { + // Only an explicitly retryable failure with remaining budget leaves + // the root open. Retries share the same attempt, so closing it early + // prevents a successful retry from finalizing it as done (#4326). + // Terminal failures must close it even when retry budget is unused: + // callers can record a permanent failure and acknowledge the task. + // + // Optional downstream stages (summary/question/wiki/graph) do not + // poison the attempt: they can fail without invalidating the parsed + // document. + if isMainPipelineStage(span.Name) && !willRetry { t.FinalizeAttempt(ctx, span.KnowledgeID, span.Attempt, types.SpanStatusFailed, nil, errorCode, errorMessage) } @@ -725,12 +746,28 @@ func contains(haystack []string, needle string) bool { return false } +// attemptStillHasRetry reports whether retry budget remains, not whether the +// caller will actually request a retry. It reads ProcessDocument's isLastRetry +// counters: the Lite executor's injected metadata wins, otherwise +// the asynq counters. Outside a task runner (a synchronous reparse, a test +// harness) neither is present and the failure is final. +func attemptStillHasRetry(ctx context.Context) bool { + if retried, limit, ok := types.TaskRetryMetadataFromContext(ctx); ok { + return retried < limit + } + retryCount, ok := asynq.GetRetryCount(ctx) + if !ok { + return false + } + maxRetry, ok := asynq.GetMaxRetry(ctx) + return ok && retryCount < maxRetry +} + // isMainPipelineStage reports whether stage is one of the 5 mandatory // pipeline stages (docreader / chunking / embedding / multimodal / -// postprocess). A failure in any of these terminally invalidates the -// attempt and must close the root as failed. Optional downstream stages -// added later (summary, question, wiki, graph) do NOT match — those -// can fail individually without poisoning the parse result. +// postprocess). A terminal failure in any of these closes the root. Optional +// downstream stages added later (summary, question, wiki, graph) do NOT match +// — those can fail individually without poisoning the parse result. func isMainPipelineStage(name string) bool { for _, s := range types.AllStages { if s == name { @@ -884,10 +921,11 @@ func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string, func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span { return nil } -func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {} -func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {} -func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {} -func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil } +func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {} +func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {} +func (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {} +func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {} +func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil } func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span { return nil } diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go index ba732fb3..9a4c6e15 100644 --- a/internal/application/service/knowledge_span_tracker_test.go +++ b/internal/application/service/knowledge_span_tracker_test.go @@ -537,3 +537,100 @@ func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) { "kid-reparse").Row().Scan(&newStatus)) assert.Equal(t, types.SpanStatusRunning, newStatus) } + +// rootSpan reads the attempt's root span: the status the UI's error banner and +// its progress bar are both driven by. +func rootSpan(t *testing.T, db *gorm.DB, knowledgeID string, attempt int) (status, code, message string) { + t.Helper() + require.NoError(t, db.Raw( + `SELECT status, COALESCE(error_code, ''), COALESCE(error_message, '') + FROM knowledge_processing_spans + WHERE knowledge_id = ? AND attempt = ? AND kind = 'root'`, + knowledgeID, attempt).Row().Scan(&status, &code, &message)) + return status, code, message +} + +func TestSpanTracker_FailSpanRetryable_KeepsRootOpenWhileRetriesRemain(t *testing.T) { + ctx := context.Background() + // Every main pipeline stage goes through the same deferral, not just the + // docreader one a report happens to show. + for _, stage := range types.AllStages { + t.Run(stage, func(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + kid := "kid-retry-" + stage + _, attempt, err := tracker.OpenAttempt(ctx, kid, "") + require.NoError(t, err) + + // The first run fails in a main stage. asynq retries reuse the + // payload, so the next run records onto this same attempt and may + // still complete it. + first := types.WithTaskRetryMetadata(ctx, 0, 3) + span := tracker.BeginStage(first, kid, attempt, stage, nil) + tracker.FailSpanRetryable(first, span, "STAGE_FAILED", "transient upstream", errors.New("boom")) + + status, code, _ := rootSpan(t, db, kid, attempt) + assert.Equal(t, types.SpanStatusRunning, status, "root stays open while a retry is left") + assert.Empty(t, code, "an attempt that can still succeed must not be attributed a failure") + + // The retry succeeds. Before this fix the root was already terminal + // here, so FinalizeAttempt returned on its guard and the UI showed + // a red error banner above an all-done trace. + tracker.FinalizeAttempt(types.WithTaskRetryMetadata(ctx, 1, 3), + kid, attempt, types.SpanStatusDone, nil, "", "") + status, code, message := rootSpan(t, db, kid, attempt) + assert.Equal(t, types.SpanStatusDone, status) + assert.Empty(t, code, "no stale error code survives a successful retry") + assert.Empty(t, message) + + // A failure arriving after the attempt closed must not reopen it. + late := tracker.BeginStage(first, kid, attempt, stage, nil) + tracker.FailSpanRetryable(types.WithTaskRetryMetadata(ctx, 3, 3), + late, "STAGE_FAILED", "later failure", errors.New("boom")) + status, _, _ = rootSpan(t, db, kid, attempt) + assert.Equal(t, types.SpanStatusDone, status, "a terminal attempt stays terminal") + }) + } +} + +func TestSpanTracker_FailSpanRetryable_ClosesRootOnTheLastRetry(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := context.Background() + + _, attempt, err := tracker.OpenAttempt(ctx, "kid-final", "") + require.NoError(t, err) + last := types.WithTaskRetryMetadata(ctx, 3, 3) + docreader := tracker.BeginStage(last, "kid-final", attempt, types.StageDocReader, nil) + tracker.FailSpanRetryable(last, docreader, + "DOCREADER_PARSE_FAILED", "engine core initialization failed", errors.New("boom")) + status, code, message := rootSpan(t, db, "kid-final", attempt) + assert.Equal(t, types.SpanStatusFailed, status, "exhausted retries must not hang at 进行中") + assert.Equal(t, "DOCREADER_PARSE_FAILED", code) + assert.Equal(t, "engine core initialization failed", message) + + // With no task runner in the context at all — a synchronous reparse, or a + // caller outside asynq — the failure is final, as it was before. + _, syncAttempt, err := tracker.OpenAttempt(ctx, "kid-sync", "") + require.NoError(t, err) + post := tracker.BeginStage(ctx, "kid-sync", syncAttempt, types.StagePostProcess, nil) + tracker.FailSpanRetryable(ctx, post, "POSTPROCESS_FAILED", "no retry metadata", errors.New("boom")) + status, _, _ = rootSpan(t, db, "kid-sync", syncAttempt) + assert.Equal(t, types.SpanStatusFailed, status) +} + +// Unused queue budget does not make an acknowledged failure retryable. +func TestSpanTracker_FailSpan_ClosesRootWithUnusedRetries(t *testing.T) { + for _, stage := range types.AllStages { + t.Run(stage, func(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3) + _, attempt, err := tracker.OpenAttempt(ctx, "terminal", "") + require.NoError(t, err) + span := tracker.BeginStage(ctx, "terminal", attempt, stage, nil) + tracker.FailSpan(ctx, span, "PERMANENT_FAILURE", "cannot continue", nil) + status, code, message := rootSpan(t, db, "terminal", attempt) + require.Equal(t, types.SpanStatusFailed, status) + require.Equal(t, "PERMANENT_FAILURE", code) + require.Equal(t, "cannot continue", message) + }) + } +}