fix(knowledge): enqueue stale summary refreshes

main
wizardchen 2026-07-28 16:04:50 +08:00 committed by lyingbug
parent 1320c50bde
commit 1b9858ecb8
11 changed files with 253 additions and 30 deletions

View File

@ -122,10 +122,13 @@ const saveMetadata = async () => {
value[key] = parseMetadataValue(row);
}
metadataSaving.value = true;
await updateKnowledgeMetadata(props.details.id, value);
const result: any = await updateKnowledgeMetadata(props.details.id, value);
props.details.custom_metadata = value;
metadataEditing.value = false;
props.details.summary_status = props.details.description ? 'pending' : props.details.summary_status;
if (result?.data) {
props.details.summary_status = result.data.summary_status || props.details.summary_status;
props.details.description = result.data.description ?? props.details.description;
}
MessagePlugin.success(t('common.saveSuccess'));
} catch (error: any) {
MessagePlugin.error(error?.message || t('common.saveFailed'));
@ -1026,7 +1029,8 @@ const saveChunkEdit = async (item: any) => {
});
Object.assign(item, result.data);
editingChunkId.value = '';
props.details.summary_status = props.details.description ? 'pending' : props.details.summary_status;
props.details.summary_status = result.summary_status || props.details.summary_status;
props.details.description = result.description ?? props.details.description;
MessagePlugin.success(t('common.saveSuccess'));
} catch (error: any) {
MessagePlugin.error(error?.message || t('common.saveFailed'));
@ -1043,6 +1047,8 @@ const toggleChunkEnabled = async (item: any, isEnabled: boolean) => {
expected_revision: item.content_revision || 0,
});
Object.assign(item, result.data);
props.details.summary_status = result.summary_status || props.details.summary_status;
props.details.description = result.description ?? props.details.description;
} catch (error: any) {
MessagePlugin.error(error?.message || t('common.error'));
} finally {
@ -1142,6 +1148,8 @@ const revertChunk = async (item: any, revision: number) => {
try {
const result: any = await revertDocumentChunk(props.details.id, item.id, revision, item.content_revision || 0);
Object.assign(item, result.data);
props.details.summary_status = result.summary_status || props.details.summary_status;
props.details.description = result.description ?? props.details.description;
delete chunkHistories.value[item.id];
const historyResult: any = await listChunkRevisions(props.details.id, item.id);
chunkHistories.value[item.id] = historyResult?.data || [];

View File

@ -31,6 +31,8 @@ type chunkService struct {
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
task interfaces.TaskEnqueuer
spanTracker SpanTracker
}
// NewChunkService creates a new chunk service
@ -47,6 +49,8 @@ func NewChunkService(
modelService interfaces.ModelService,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
task interfaces.TaskEnqueuer,
spanTracker SpanTracker,
) interfaces.ChunkService {
return &chunkService{
chunkRepository: chunkRepository,
@ -55,6 +59,8 @@ func NewChunkService(
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
task: task,
spanTracker: spanTracker,
}
}
@ -457,10 +463,14 @@ func (s *chunkService) UpdateDocumentChunk(
logger.Warnf(ctx, "Failed to rebuild parent chunk after edit: %v", err)
}
}
if bodyChanged {
if bodyChanged || newEnabled != revision.IsEnabled {
knowledge, getErr := s.knowledgeRepo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID)
if getErr == nil && knowledge.SummaryStatus == types.SummaryStatusCompleted {
_ = s.knowledgeRepo.UpdateKnowledgeColumn(ctx, knowledge.ID, "summary_status", types.SummaryStatusPending)
if getErr == nil {
if err := enqueueSummaryRefresh(
ctx, s.knowledgeRepo, s.task, s.kbRepository, s.spanTracker, knowledge,
); err != nil {
logger.Warnf(ctx, "Chunk saved but summary refresh enqueue failed for %s: %v", knowledge.ID, err)
}
}
}
if err := s.syncChunkIndex(ctx, chunk); err != nil {

View File

@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"reflect"
"strings"
"sync"
"time"
@ -610,8 +611,8 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types
if knowledge.Description != "" {
record.Description = knowledge.Description
}
metadataChanged := knowledge.CustomMetadata != nil
if metadataChanged {
metadataChanged := false
if knowledge.CustomMetadata != nil {
var custom map[string]interface{}
if err := json.Unmarshal(knowledge.CustomMetadata, &custom); err != nil {
return fmt.Errorf("custom_metadata must be a JSON object: %w", err)
@ -629,6 +630,11 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types
return fmt.Errorf("custom_metadata field %q must be a string, number, boolean, or null", key)
}
}
existing := make(map[string]interface{})
if len(record.CustomMetadata) > 0 {
_ = json.Unmarshal(record.CustomMetadata, &existing)
}
metadataChanged = !reflect.DeepEqual(existing, custom)
record.CustomMetadata = knowledge.CustomMetadata
}
@ -637,9 +643,11 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types
logger.Errorf(ctx, "Failed to update knowledge: %v", err)
return err
}
if metadataChanged {
if record.SummaryStatus == types.SummaryStatusCompleted {
_ = s.repo.UpdateKnowledgeColumn(ctx, record.ID, "summary_status", types.SummaryStatusPending)
if metadataChanged && record.SummaryStatus != "" && record.SummaryStatus != types.SummaryStatusNone {
if err := enqueueSummaryRefresh(ctx, s.repo, s.task, s.kbService, s.tracker(), record); err != nil {
logger.Warnf(ctx, "Metadata saved but summary refresh enqueue failed for %s: %v", record.ID, err)
} else {
logger.Infof(ctx, "Enqueued summary refresh after metadata update, knowledge ID: %s", record.ID)
}
}
logger.Infof(ctx, "Knowledge updated successfully, ID: %s", knowledge.ID)

View File

@ -2,10 +2,13 @@ package service
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/require"
)
@ -64,6 +67,31 @@ type metadataUpdateChunkRepo struct {
listCalls int
}
type metadataUpdateKBService struct {
interfaces.KnowledgeBaseService
}
func (metadataUpdateKBService) GetKnowledgeBaseByID(
_ context.Context, id string,
) (*types.KnowledgeBase, error) {
return &types.KnowledgeBase{ID: id, SummaryModelID: "summary-model"}, nil
}
type metadataUpdateTaskEnqueuer struct {
tasks []*asynq.Task
err error
}
func (e *metadataUpdateTaskEnqueuer) Enqueue(
task *asynq.Task, _ ...asynq.Option,
) (*asynq.TaskInfo, error) {
if e.err != nil {
return nil, e.err
}
e.tasks = append(e.tasks, task)
return nil, nil
}
func (r *metadataUpdateChunkRepo) ListChunksByKnowledgeID(
_ context.Context, _ uint64, _ string,
) ([]*types.Chunk, error) {
@ -74,13 +102,17 @@ func (r *metadataUpdateChunkRepo) ListChunksByKnowledgeID(
func TestUpdateKnowledgeMetadataDoesNotReindexChunks(t *testing.T) {
t.Parallel()
repo := &metadataUpdateKnowledgeRepo{knowledge: &types.Knowledge{
ID: "knowledge-1",
TenantID: 7,
SummaryStatus: types.SummaryStatusCompleted,
CustomMetadata: types.JSON(`{"region":"Beijing"}`),
ID: "knowledge-1",
TenantID: 7,
KnowledgeBaseID: "kb-1",
SummaryStatus: types.SummaryStatusCompleted,
CustomMetadata: types.JSON(`{"region":"Beijing"}`),
}}
chunkRepo := &metadataUpdateChunkRepo{}
service := &knowledgeService{repo: repo, chunkRepo: chunkRepo}
taskEnqueuer := &metadataUpdateTaskEnqueuer{}
service := &knowledgeService{
repo: repo, chunkRepo: chunkRepo, kbService: metadataUpdateKBService{}, task: taskEnqueuer,
}
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
err := service.UpdateKnowledge(ctx, &types.Knowledge{
@ -92,4 +124,49 @@ func TestUpdateKnowledgeMetadataDoesNotReindexChunks(t *testing.T) {
require.JSONEq(t, `{"region":"Shanghai"}`, string(repo.knowledge.CustomMetadata))
require.Zero(t, chunkRepo.listCalls)
require.Equal(t, types.SummaryStatusPending, repo.summaryStatusUpdate)
require.Len(t, taskEnqueuer.tasks, 1)
require.Equal(t, types.TypeSummaryGeneration, taskEnqueuer.tasks[0].Type())
var payload types.SummaryGenerationPayload
require.NoError(t, json.Unmarshal(taskEnqueuer.tasks[0].Payload(), &payload))
require.True(t, payload.Refresh)
require.Equal(t, "knowledge-1", payload.KnowledgeID)
}
func TestUpdateKnowledgeMetadataDoesNotLeaveTasklessPendingStatus(t *testing.T) {
t.Parallel()
repo := &metadataUpdateKnowledgeRepo{knowledge: &types.Knowledge{
ID: "knowledge-1", TenantID: 7, KnowledgeBaseID: "kb-1",
SummaryStatus: types.SummaryStatusCompleted,
}}
service := &knowledgeService{
repo: repo, chunkRepo: &metadataUpdateChunkRepo{}, kbService: metadataUpdateKBService{},
task: &metadataUpdateTaskEnqueuer{err: errors.New("queue unavailable")},
}
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
require.NoError(t, service.UpdateKnowledge(ctx, &types.Knowledge{
ID: "knowledge-1", CustomMetadata: types.JSON(`{"region":"Shanghai"}`),
}))
require.Equal(t, types.SummaryStatusFailed, repo.summaryStatusUpdate)
}
func TestUpdateKnowledgeUnchangedMetadataDoesNotRefreshSummary(t *testing.T) {
t.Parallel()
repo := &metadataUpdateKnowledgeRepo{knowledge: &types.Knowledge{
ID: "knowledge-1", TenantID: 7, KnowledgeBaseID: "kb-1",
SummaryStatus: types.SummaryStatusCompleted,
CustomMetadata: types.JSON(`{"department":"R&D","region":"Shanghai"}`),
}}
taskEnqueuer := &metadataUpdateTaskEnqueuer{}
service := &knowledgeService{
repo: repo, chunkRepo: &metadataUpdateChunkRepo{}, kbService: metadataUpdateKBService{}, task: taskEnqueuer,
}
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
require.NoError(t, service.UpdateKnowledge(ctx, &types.Knowledge{
ID: "knowledge-1",
CustomMetadata: types.JSON(`{"region":"Shanghai","department":"R&D"}`),
}))
require.Empty(t, taskEnqueuer.tasks)
require.Nil(t, repo.summaryStatusUpdate)
}

View File

@ -304,6 +304,11 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
enqueuedQuestionCount := 0
if willSpawnSummary {
enqueuedSummary = s.enqueueSummaryGenerationTask(ctx, payload, attempt)
if !enqueuedSummary {
_ = s.knowledgeRepo.UpdateKnowledgeColumn(
ctx, payload.KnowledgeID, "summary_status", types.SummaryStatusFailed,
)
}
if willSpawnQuestion {
// Create the postprocess.question grouping span up front so the
// per-batch subspans (enqueued just below, run later in their own

View File

@ -942,6 +942,16 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
payload.Attempt, payload.KnowledgeID)
return nil
}
if payload.Refresh {
_, err := s.RegenerateKnowledgeSummary(ctx, payload.KnowledgeID)
if err != nil {
// A retry will move the row back to processing. Keeping failed here
// ensures both Redis workers and the Lite executor have a terminal,
// non-pending state if every retry is exhausted.
_ = s.repo.UpdateKnowledgeColumn(ctx, payload.KnowledgeID, "summary_status", types.SummaryStatusFailed)
}
return err
}
// Open a subspan under the parent attempt's postprocess stage so the
// trace surface shows the real summary-generation duration (LLM call
@ -991,6 +1001,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
if kb.SummaryModelID == "" {
logger.Warn(ctx, "Knowledge base summary model ID is empty, skipping summary generation")
_ = s.repo.UpdateKnowledgeColumn(ctx, payload.KnowledgeID, "summary_status", types.SummaryStatusFailed)
summaryOut["skipped"] = "no_summary_model"
return nil
}
@ -1127,7 +1138,6 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
}
if staleSummary {
logger.Infof(ctx, "Discarding stale summary for knowledge %s", payload.KnowledgeID)
_ = s.repo.UpdateKnowledgeColumn(ctx, payload.KnowledgeID, "summary_status", types.SummaryStatusPending)
summaryOut["skipped"] = "content_revision_changed"
return nil
}
@ -2107,8 +2117,8 @@ func (s *knowledgeService) RegenerateChunkQuestions(
}
// RegenerateKnowledgeSummary refreshes both the knowledge description and the
// summary chunk(s), then reindexes all retrieval artifacts with current
// custom metadata. It is intentionally idempotent.
// summary chunk(s), then reindexes only those summary chunks. It is
// intentionally idempotent.
func (s *knowledgeService) RegenerateKnowledgeSummary(
ctx context.Context, knowledgeID string,
) (*types.Knowledge, error) {
@ -2154,13 +2164,13 @@ func (s *knowledgeService) RegenerateKnowledgeSummary(
}
latestKnowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil || string(latestKnowledge.CustomMetadata) != metadataVersion {
_ = s.repo.UpdateKnowledgeColumn(ctx, knowledgeID, "summary_status", types.SummaryStatusPending)
_ = s.repo.UpdateKnowledgeColumn(ctx, knowledgeID, "summary_status", types.SummaryStatusFailed)
return nil, ErrChunkRevisionConflict
}
for _, sourceChunk := range textChunks {
latestChunk, getErr := s.chunkRepo.GetChunkByID(ctx, tenantID, sourceChunk.ID)
if getErr != nil || latestChunk.ContentRevision != sourceChunk.ContentRevision || latestChunk.IsEnabled != sourceChunk.IsEnabled {
_ = s.repo.UpdateKnowledgeColumn(ctx, knowledgeID, "summary_status", types.SummaryStatusPending)
_ = s.repo.UpdateKnowledgeColumn(ctx, knowledgeID, "summary_status", types.SummaryStatusFailed)
return nil, ErrChunkRevisionConflict
}
}
@ -2173,6 +2183,7 @@ func (s *knowledgeService) RegenerateKnowledgeSummary(
if kb.NeedsEmbeddingModel() {
found := false
maxIndex := 0
summaryChunks := make([]*types.Chunk, 0, 1)
for _, chunk := range allChunks {
if chunk.ChunkIndex > maxIndex {
maxIndex = chunk.ChunkIndex
@ -2185,6 +2196,7 @@ func (s *knowledgeService) RegenerateKnowledgeSummary(
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
return nil, err
}
summaryChunks = append(summaryChunks, chunk)
found = true
}
}
@ -2198,9 +2210,9 @@ func (s *knowledgeService) RegenerateKnowledgeSummary(
if err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
return nil, err
}
allChunks = append(allChunks, summaryChunk)
summaryChunks = append(summaryChunks, summaryChunk)
}
if err := s.updateChunkVector(ctx, knowledge.KnowledgeBaseID, allChunks); err != nil {
if err := s.updateChunkVector(ctx, knowledge.KnowledgeBaseID, summaryChunks); err != nil {
return nil, err
}
}

View File

@ -482,12 +482,14 @@ func TestSummaryQuestionPayload_AttemptRoundTrip(t *testing.T) {
KnowledgeID: "kid-7",
Language: "zh-CN",
Attempt: 5,
Refresh: true,
}
sumBytes, err := json.Marshal(sumIn)
require.NoError(t, err)
var sumOut types.SummaryGenerationPayload
require.NoError(t, json.Unmarshal(sumBytes, &sumOut))
assert.Equal(t, 5, sumOut.Attempt)
assert.True(t, sumOut.Refresh)
qIn := types.QuestionGenerationPayload{
TenantID: 42,

View File

@ -0,0 +1,77 @@
package service
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
)
type summaryKnowledgeBaseReader interface {
GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error)
}
// enqueueSummaryRefresh marks an existing summary as queued and enqueues an
// independent refresh task. The pending status is written before Enqueue
// because the Lite executor may run the task synchronously inside Enqueue.
func enqueueSummaryRefresh(
ctx context.Context,
repo interfaces.KnowledgeRepository,
taskEnqueuer interfaces.TaskEnqueuer,
kbReader summaryKnowledgeBaseReader,
tracker SpanTracker,
knowledge *types.Knowledge,
) error {
if knowledge == nil || knowledge.SummaryStatus == "" || knowledge.SummaryStatus == types.SummaryStatusNone {
return nil
}
markFailed := func() {
if repo != nil {
_ = repo.UpdateKnowledgeColumn(ctx, knowledge.ID, "summary_status", types.SummaryStatusFailed)
}
}
if repo == nil || taskEnqueuer == nil || kbReader == nil {
markFailed()
return fmt.Errorf("summary refresh dependencies are unavailable")
}
kb, err := kbReader.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
if err != nil {
markFailed()
return err
}
if kb.SummaryModelID == "" {
markFailed()
return fmt.Errorf("summary model is not configured")
}
if tracker == nil {
tracker = noopSpanTracker{}
}
language, _ := types.LanguageFromContext(ctx)
payload := types.SummaryGenerationPayload{
TenantID: knowledge.TenantID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
KnowledgeID: knowledge.ID,
Language: language,
Attempt: tracker.LatestAttempt(ctx, knowledge.ID),
Refresh: true,
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
markFailed()
return err
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes,
asynq.Queue(types.QueueSummary), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
_ = repo.UpdateKnowledgeColumn(ctx, knowledge.ID, "summary_status", types.SummaryStatusPending)
if _, err = taskEnqueuer.Enqueue(task); err != nil {
markFailed()
return err
}
return nil
}

View File

@ -237,10 +237,16 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
logger.Infof(ctx, "Knowledge chunk updated successfully, knowledge ID: %s, chunk ID: %s",
secutils.SanitizeForLog(knowledgeID), secutils.SanitizeForLog(chunk.ID))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": chunk,
})
knowledge, getErr := h.kgService.GetKnowledgeByID(ctx, knowledgeID)
if getErr != nil {
logger.Warnf(ctx, "Chunk updated but failed to reload summary status for %s: %v", knowledgeID, getErr)
}
response := gin.H{"success": true, "data": chunk}
if knowledge != nil {
response["summary_status"] = knowledge.SummaryStatus
response["description"] = knowledge.Description
}
c.JSON(http.StatusOK, response)
}
func (h *ChunkHandler) ListChunkRevisions(c *gin.Context) {
@ -263,7 +269,7 @@ type RevertChunkRequest struct {
}
func (h *ChunkHandler) RevertChunk(c *gin.Context) {
chunk, _, err := h.fetchChunkAndVerifyOwnership(c)
chunk, knowledgeID, err := h.fetchChunkAndVerifyOwnership(c)
if err != nil {
c.Error(err)
return
@ -286,7 +292,16 @@ func (h *ChunkHandler) RevertChunk(c *gin.Context) {
c.Error(errors.NewBadRequestError(err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": updated})
knowledge, getErr := h.kgService.GetKnowledgeByID(c.Request.Context(), knowledgeID)
if getErr != nil {
logger.Warnf(c.Request.Context(), "Chunk reverted but failed to reload summary status for %s: %v", knowledgeID, getErr)
}
response := gin.H{"success": true, "data": updated}
if knowledge != nil {
response["summary_status"] = knowledge.SummaryStatus
response["description"] = knowledge.Description
}
c.JSON(http.StatusOK, response)
}
type UpsertGeneratedQuestionRequest struct {

View File

@ -1565,11 +1565,16 @@ func (h *KnowledgeHandler) UpdateKnowledge(c *gin.Context) {
c.Error(errors.NewInternalServerError(err.Error()))
return
}
updated, getErr := h.kgService.GetKnowledgeByID(effCtx, id)
if getErr != nil {
logger.Warnf(ctx, "Knowledge updated but failed to reload status for %s: %v", id, getErr)
}
logger.Infof(ctx, "Knowledge updated successfully, knowledge ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge chunk updated successfully",
"message": "Knowledge updated successfully",
"data": updated,
})
}

View File

@ -360,6 +360,10 @@ type SummaryGenerationPayload struct {
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
Language string `json:"language,omitempty"`
// Refresh marks an independently queued refresh after document metadata or
// chunk edits. Refresh tasks are not part of the parse attempt's pending
// subtask counter and update existing summary chunks in place.
Refresh bool `json:"refresh,omitempty"`
// Attempt links this task to the parent parse attempt so the worker
// can record a postprocess.summary subspan under the right attempt's
// postprocess stage. See QuestionGenerationPayload.Attempt notes.