diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 687ff471..065912c6 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" @@ -53,6 +54,7 @@ func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository { // CreateKnowledge creates knowledge func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { + knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage) err := r.db.WithContext(ctx).Create(knowledge).Error return err } @@ -306,6 +308,7 @@ func (r *knowledgeRepository) RenameKnowledgeFolderPath( // UpdateKnowledge updates knowledge func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { + knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage) omit := omitFieldsOnUpdate // Legacy/unit-test schemas created before custom_metadata should continue // to support unrelated updates when the caller did not provide the field. @@ -321,6 +324,11 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledg if len(knowledgeList) == 0 { return nil } + for _, knowledge := range knowledgeList { + if knowledge != nil { + knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage) + } + } return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error } @@ -513,6 +521,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn( column string, value interface{}, ) error { + if column == "error_message" { + switch v := value.(type) { + case string: + value = common.CleanInvalidUTF8(v) + case []byte: + value = common.CleanInvalidUTF8(string(v)) + } + } err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error return err } @@ -529,6 +545,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns( if len(values) == 0 { return nil } + if value, ok := values["error_message"]; ok { + switch v := value.(type) { + case string: + values["error_message"] = common.CleanInvalidUTF8(v) + case []byte: + values["error_message"] = common.CleanInvalidUTF8(string(v)) + } + } return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error } diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.go index b27efd1b..abcb68a9 100644 --- a/internal/application/repository/knowledge_finalize_test.go +++ b/internal/application/repository/knowledge_finalize_test.go @@ -5,6 +5,7 @@ import ( "sync" "sync/atomic" "testing" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" @@ -99,6 +100,23 @@ func reloadKnowledgeErrorMessage(t *testing.T, db *gorm.DB, id string) string { return msg } +func TestKnowledgeRepository_UpdateKnowledgeColumnsSanitizesErrorMessage(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db) + id := insertProcessingKnowledge(t, db) + invalid := "parse failed " + string([]byte{0xef, 0xbc, 0x2e}) + + require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), id, map[string]interface{}{ + "error_message": invalid, + })) + + got := reloadKnowledgeErrorMessage(t, db, id) + if !utf8.ValidString(got) { + t.Fatalf("persisted error_message is invalid UTF-8: % x", []byte(got)) + } + assert.Equal(t, "parse failed .", got) +} + func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string { t.Helper() id := uuid.New().String() diff --git a/internal/application/repository/knowledge_span_repo.go b/internal/application/repository/knowledge_span_repo.go index 1defb0f7..c610f1e9 100644 --- a/internal/application/repository/knowledge_span_repo.go +++ b/internal/application/repository/knowledge_span_repo.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/types" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -61,6 +62,12 @@ func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.Knowled if row.Attempt == 0 { row.Attempt = 1 } + // Error fields may originate from third-party parser/model responses. + // Normalize them at the persistence boundary so SQLite/PostgreSQL and the + // tracing API never receive malformed UTF-8. + row.ErrorCode = common.CleanInvalidUTF8(row.ErrorCode) + row.ErrorMessage = common.CleanInvalidUTF8(row.ErrorMessage) + row.ErrorDetail = common.CleanInvalidUTF8(row.ErrorDetail) // We let GORM populate created_at/updated_at via the autoCreate / // autoUpdate tags. ON CONFLICT updates only the fields that may // transition between calls — name/kind/parent are immutable once @@ -163,6 +170,7 @@ func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID strin // Postgres-specific WITH RECURSIVE would be denser but harder to test on // the SQLite Lite backend. The iterative path stays portable. func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) { + reason = common.CleanInvalidUTF8(reason) frontier := []string{parentSpanID} var totalAffected int64 for depth := 0; depth < 16 && len(frontier) > 0; depth++ { @@ -215,6 +223,8 @@ func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowled func (r *knowledgeSpanRepository) CancelAllOpenSpans( ctx context.Context, knowledgeID string, attempt int, errorCode, reason string, ) (int64, error) { + errorCode = common.CleanInvalidUTF8(errorCode) + reason = common.CleanInvalidUTF8(reason) now := time.Now() updates := map[string]any{ "status": types.SpanStatusCancelled, @@ -240,6 +250,8 @@ func (r *knowledgeSpanRepository) CancelOpenSpansByName( if knowledgeID == "" || attempt <= 0 || name == "" { return 0, nil } + errorCode = common.CleanInvalidUTF8(errorCode) + reason = common.CleanInvalidUTF8(reason) now := time.Now() res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}). Where("knowledge_id = ? AND attempt = ? AND name = ? AND status IN ?", diff --git a/internal/application/repository/knowledge_span_repo_test.go b/internal/application/repository/knowledge_span_repo_test.go index 341a3d4f..72b70ade 100644 --- a/internal/application/repository/knowledge_span_repo_test.go +++ b/internal/application/repository/knowledge_span_repo_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" "time" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" "github.com/stretchr/testify/assert" @@ -85,6 +86,37 @@ func TestKnowledgeSpanRepo_UpsertAndList(t *testing.T) { assert.Equal(t, int64(2000), rows[0].DurationMs) } +func TestKnowledgeSpanRepo_UpsertSanitizesErrorFields(t *testing.T) { + repo, _ := setupSpanTestRepo(t) + now := time.Now() + invalid := "prefix " + string([]byte{0xef, 0xbc, 0x2e}) + " suffix" + + require.NoError(t, repo.Upsert(context.Background(), &types.KnowledgeProcessingSpan{ + KnowledgeID: "kid-invalid-utf8", + Attempt: 1, + SpanID: "span-invalid", + Name: types.StageDocReader, + Kind: types.SpanKindStage, + Status: types.SpanStatusFailed, + ErrorCode: invalid, + ErrorMessage: invalid, + ErrorDetail: invalid, + StartedAt: &now, + })) + + rows, err := repo.ListByAttempt(context.Background(), "kid-invalid-utf8", 1) + require.NoError(t, err) + require.Len(t, rows, 1) + for _, value := range []string{rows[0].ErrorCode, rows[0].ErrorMessage, rows[0].ErrorDetail} { + if !utf8.ValidString(value) { + t.Fatalf("persisted error field is invalid UTF-8: % x", []byte(value)) + } + if value != "prefix . suffix" { + t.Fatalf("persisted error field = %q, want %q", value, "prefix . suffix") + } + } +} + // TestKnowledgeSpanRepo_NextAttempt confirms that NextAttempt allocates // a fresh number per knowledge, isolating reparse history. Critical // because the API layer renders attempt history by this number. @@ -182,6 +214,45 @@ func TestKnowledgeSpanRepo_CancelOpenSpansByName(t *testing.T) { assert.Equal(t, types.SpanStatusRunning, statusBy["q-old"]) } +func TestKnowledgeSpanRepo_CancelPathsSanitizeErrorFields(t *testing.T) { + repo, _ := setupSpanTestRepo(t) + ctx := context.Background() + kid := "kid-cancel-invalid-utf8" + now := time.Now() + for _, r := range []*types.KnowledgeProcessingSpan{ + {KnowledgeID: kid, Attempt: 1, SpanID: "root", Name: "root", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now}, + {KnowledgeID: kid, Attempt: 1, SpanID: "descendant", ParentSpanID: "root", Name: "descendant", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now}, + {KnowledgeID: kid, Attempt: 1, SpanID: "bulk", Name: "bulk", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now}, + {KnowledgeID: kid, Attempt: 1, SpanID: "named", Name: "named", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now}, + } { + require.NoError(t, repo.Upsert(ctx, r)) + } + + invalidCode := "CODE " + string([]byte{0xef, 0xbc, 0x2e}) + invalidReason := "reason " + string([]byte{0xef, 0xbc, 0x2e}) + _, err := repo.CancelOpenSpansByName(ctx, kid, 1, "named", invalidCode, invalidReason) + require.NoError(t, err) + _, err = repo.CancelDescendants(ctx, kid, 1, "root", invalidReason) + require.NoError(t, err) + _, err = repo.CancelAllOpenSpans(ctx, kid, 1, invalidCode, invalidReason) + require.NoError(t, err) + + rows, err := repo.ListByAttempt(ctx, kid, 1) + require.NoError(t, err) + byID := make(map[string]types.KnowledgeProcessingSpan, len(rows)) + for _, row := range rows { + byID[row.SpanID] = row + } + assert.Equal(t, "UPSTREAM_FAILED", byID["descendant"].ErrorCode) + assert.Equal(t, "reason .", byID["descendant"].ErrorMessage) + for _, id := range []string{"root", "bulk", "named"} { + assert.Equal(t, "CODE .", byID[id].ErrorCode, id) + assert.Equal(t, "reason .", byID[id].ErrorMessage, id) + assert.True(t, utf8.ValidString(byID[id].ErrorCode), id) + assert.True(t, utf8.ValidString(byID[id].ErrorMessage), id) + } +} + // TestKnowledgeSpanRepo_ListAttemptIsolation guarantees that different // attempts of the same knowledge stay queryable independently — the // foundation for the "show history" UI navigation (?attempt=N). diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 6c2f7350..ce9365fc 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -11,6 +11,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/common" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/infrastructure/chunker" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" @@ -251,6 +252,23 @@ func (s *knowledgeService) processChunks(ctx context.Context, options = opts[0] } + // Parser output and manually supplied passages can contain malformed byte + // sequences. Clean them before logging, chunk persistence, or embedding; + // the embedding provider and tracing/database drivers expect valid UTF-8. + for i := range chunks { + chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content) + chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader) + for j := range chunks[i].Images { + chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL) + chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption) + chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText) + chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL) + } + } + for i := range options.ParentChunks { + options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content) + } + // Check if knowledge is being deleted/cancelled before processing. // Both statuses short-circuit identically here — there's nothing to clean // up yet so the branch is purely "stop early". @@ -2896,6 +2914,7 @@ func (s *knowledgeService) UpdateImageInfo( chunkID string, imageInfo string, ) error { + imageInfo = common.CleanInvalidUTF8(imageInfo) var images []*types.ImageInfo if err := json.Unmarshal([]byte(imageInfo), &images); err != nil { logger.Errorf(ctx, "Failed to unmarshal image info: %v", err) @@ -3516,6 +3535,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e // Step 3: Split into chunks using Go chunker. Browser textareas normalize // pasted content to LF, so normalize uploaded source text before calculating // chunk boundaries as well. + sanitizeReadResult(convertResult) convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent) chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig) @@ -3572,6 +3592,28 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } +// sanitizeReadResult protects every text field that can cross from a parser +// into the embedding, storage, or tracing layers. A parser may return a Go +// string containing arbitrary bytes even though the string type itself does +// not enforce UTF-8 validity. +func sanitizeReadResult(result *types.ReadResult) { + if result == nil { + return + } + result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent) + result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath) + result.Error = common.CleanInvalidUTF8(result.Error) + for key, value := range result.Metadata { + result.Metadata[key] = common.CleanInvalidUTF8(value) + } + for i := range result.ImageRefs { + result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename) + result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef) + result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType) + result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey) + } +} + // convert handles both file and URL reading using a unified ReadRequest. func (s *knowledgeService) convert( ctx context.Context, @@ -3689,6 +3731,7 @@ func (s *knowledgeService) convert( code, "document read failed", err) return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err) } + sanitizeReadResult(result) if result.Error != "" { logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s", kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error) diff --git a/internal/common/tools_test.go b/internal/common/tools_test.go index d3a2e6e2..53ec83d8 100644 --- a/internal/common/tools_test.go +++ b/internal/common/tools_test.go @@ -1,6 +1,9 @@ package common -import "testing" +import ( + "testing" + "unicode/utf8" +) func TestParseLLMJsonResponse(t *testing.T) { type payload struct { @@ -39,6 +42,18 @@ func TestParseLLMJsonResponse(t *testing.T) { } } +func TestCleanInvalidUTF8(t *testing.T) { + input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail" + + got := CleanInvalidUTF8(input) + if !utf8.ValidString(got) { + t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got)) + } + if want := "valid .tail"; got != want { + t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want) + } +} + func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) { const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```" type payload struct {