fix(markdown): sanitize invalid UTF-8 and NUL bytes

main
十三分之五六七 2026-08-30 16:54:41 +08:00 committed by lyingbug
parent 26d9552642
commit 786d67609d
6 changed files with 184 additions and 1 deletions

View File

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
@ -53,6 +54,7 @@ func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository {
// CreateKnowledge creates knowledge
func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
err := r.db.WithContext(ctx).Create(knowledge).Error
return err
}
@ -306,6 +308,7 @@ func (r *knowledgeRepository) RenameKnowledgeFolderPath(
// UpdateKnowledge updates knowledge
func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
omit := omitFieldsOnUpdate
// Legacy/unit-test schemas created before custom_metadata should continue
// to support unrelated updates when the caller did not provide the field.
@ -321,6 +324,11 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledg
if len(knowledgeList) == 0 {
return nil
}
for _, knowledge := range knowledgeList {
if knowledge != nil {
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
}
}
return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error
}
@ -513,6 +521,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(
column string,
value interface{},
) error {
if column == "error_message" {
switch v := value.(type) {
case string:
value = common.CleanInvalidUTF8(v)
case []byte:
value = common.CleanInvalidUTF8(string(v))
}
}
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error
return err
}
@ -529,6 +545,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(
if len(values) == 0 {
return nil
}
if value, ok := values["error_message"]; ok {
switch v := value.(type) {
case string:
values["error_message"] = common.CleanInvalidUTF8(v)
case []byte:
values["error_message"] = common.CleanInvalidUTF8(string(v))
}
}
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
}

View File

@ -5,6 +5,7 @@ import (
"sync"
"sync/atomic"
"testing"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
@ -99,6 +100,23 @@ func reloadKnowledgeErrorMessage(t *testing.T, db *gorm.DB, id string) string {
return msg
}
func TestKnowledgeRepository_UpdateKnowledgeColumnsSanitizesErrorMessage(t *testing.T) {
db := setupKnowledgeTestDB(t)
repo := NewKnowledgeRepository(db)
id := insertProcessingKnowledge(t, db)
invalid := "parse failed " + string([]byte{0xef, 0xbc, 0x2e})
require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), id, map[string]interface{}{
"error_message": invalid,
}))
got := reloadKnowledgeErrorMessage(t, db, id)
if !utf8.ValidString(got) {
t.Fatalf("persisted error_message is invalid UTF-8: % x", []byte(got))
}
assert.Equal(t, "parse failed .", got)
}
func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {
t.Helper()
id := uuid.New().String()

View File

@ -5,6 +5,7 @@ import (
"errors"
"time"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@ -61,6 +62,12 @@ func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.Knowled
if row.Attempt == 0 {
row.Attempt = 1
}
// Error fields may originate from third-party parser/model responses.
// Normalize them at the persistence boundary so SQLite/PostgreSQL and the
// tracing API never receive malformed UTF-8.
row.ErrorCode = common.CleanInvalidUTF8(row.ErrorCode)
row.ErrorMessage = common.CleanInvalidUTF8(row.ErrorMessage)
row.ErrorDetail = common.CleanInvalidUTF8(row.ErrorDetail)
// We let GORM populate created_at/updated_at via the autoCreate /
// autoUpdate tags. ON CONFLICT updates only the fields that may
// transition between calls — name/kind/parent are immutable once
@ -163,6 +170,7 @@ func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID strin
// Postgres-specific WITH RECURSIVE would be denser but harder to test on
// the SQLite Lite backend. The iterative path stays portable.
func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) {
reason = common.CleanInvalidUTF8(reason)
frontier := []string{parentSpanID}
var totalAffected int64
for depth := 0; depth < 16 && len(frontier) > 0; depth++ {
@ -215,6 +223,8 @@ func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowled
func (r *knowledgeSpanRepository) CancelAllOpenSpans(
ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,
) (int64, error) {
errorCode = common.CleanInvalidUTF8(errorCode)
reason = common.CleanInvalidUTF8(reason)
now := time.Now()
updates := map[string]any{
"status": types.SpanStatusCancelled,
@ -240,6 +250,8 @@ func (r *knowledgeSpanRepository) CancelOpenSpansByName(
if knowledgeID == "" || attempt <= 0 || name == "" {
return 0, nil
}
errorCode = common.CleanInvalidUTF8(errorCode)
reason = common.CleanInvalidUTF8(reason)
now := time.Now()
res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("knowledge_id = ? AND attempt = ? AND name = ? AND status IN ?",

View File

@ -4,6 +4,7 @@ import (
"context"
"testing"
"time"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
@ -85,6 +86,37 @@ func TestKnowledgeSpanRepo_UpsertAndList(t *testing.T) {
assert.Equal(t, int64(2000), rows[0].DurationMs)
}
func TestKnowledgeSpanRepo_UpsertSanitizesErrorFields(t *testing.T) {
repo, _ := setupSpanTestRepo(t)
now := time.Now()
invalid := "prefix " + string([]byte{0xef, 0xbc, 0x2e}) + " suffix"
require.NoError(t, repo.Upsert(context.Background(), &types.KnowledgeProcessingSpan{
KnowledgeID: "kid-invalid-utf8",
Attempt: 1,
SpanID: "span-invalid",
Name: types.StageDocReader,
Kind: types.SpanKindStage,
Status: types.SpanStatusFailed,
ErrorCode: invalid,
ErrorMessage: invalid,
ErrorDetail: invalid,
StartedAt: &now,
}))
rows, err := repo.ListByAttempt(context.Background(), "kid-invalid-utf8", 1)
require.NoError(t, err)
require.Len(t, rows, 1)
for _, value := range []string{rows[0].ErrorCode, rows[0].ErrorMessage, rows[0].ErrorDetail} {
if !utf8.ValidString(value) {
t.Fatalf("persisted error field is invalid UTF-8: % x", []byte(value))
}
if value != "prefix . suffix" {
t.Fatalf("persisted error field = %q, want %q", value, "prefix . suffix")
}
}
}
// TestKnowledgeSpanRepo_NextAttempt confirms that NextAttempt allocates
// a fresh number per knowledge, isolating reparse history. Critical
// because the API layer renders attempt history by this number.
@ -182,6 +214,45 @@ func TestKnowledgeSpanRepo_CancelOpenSpansByName(t *testing.T) {
assert.Equal(t, types.SpanStatusRunning, statusBy["q-old"])
}
func TestKnowledgeSpanRepo_CancelPathsSanitizeErrorFields(t *testing.T) {
repo, _ := setupSpanTestRepo(t)
ctx := context.Background()
kid := "kid-cancel-invalid-utf8"
now := time.Now()
for _, r := range []*types.KnowledgeProcessingSpan{
{KnowledgeID: kid, Attempt: 1, SpanID: "root", Name: "root", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now},
{KnowledgeID: kid, Attempt: 1, SpanID: "descendant", ParentSpanID: "root", Name: "descendant", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
{KnowledgeID: kid, Attempt: 1, SpanID: "bulk", Name: "bulk", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
{KnowledgeID: kid, Attempt: 1, SpanID: "named", Name: "named", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
} {
require.NoError(t, repo.Upsert(ctx, r))
}
invalidCode := "CODE " + string([]byte{0xef, 0xbc, 0x2e})
invalidReason := "reason " + string([]byte{0xef, 0xbc, 0x2e})
_, err := repo.CancelOpenSpansByName(ctx, kid, 1, "named", invalidCode, invalidReason)
require.NoError(t, err)
_, err = repo.CancelDescendants(ctx, kid, 1, "root", invalidReason)
require.NoError(t, err)
_, err = repo.CancelAllOpenSpans(ctx, kid, 1, invalidCode, invalidReason)
require.NoError(t, err)
rows, err := repo.ListByAttempt(ctx, kid, 1)
require.NoError(t, err)
byID := make(map[string]types.KnowledgeProcessingSpan, len(rows))
for _, row := range rows {
byID[row.SpanID] = row
}
assert.Equal(t, "UPSTREAM_FAILED", byID["descendant"].ErrorCode)
assert.Equal(t, "reason .", byID["descendant"].ErrorMessage)
for _, id := range []string{"root", "bulk", "named"} {
assert.Equal(t, "CODE .", byID[id].ErrorCode, id)
assert.Equal(t, "reason .", byID[id].ErrorMessage, id)
assert.True(t, utf8.ValidString(byID[id].ErrorCode), id)
assert.True(t, utf8.ValidString(byID[id].ErrorMessage), id)
}
}
// TestKnowledgeSpanRepo_ListAttemptIsolation guarantees that different
// attempts of the same knowledge stay queryable independently — the
// foundation for the "show history" UI navigation (?attempt=N).

View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/common"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
@ -251,6 +252,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,
options = opts[0]
}
// Parser output and manually supplied passages can contain malformed byte
// sequences. Clean them before logging, chunk persistence, or embedding;
// the embedding provider and tracing/database drivers expect valid UTF-8.
for i := range chunks {
chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content)
chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader)
for j := range chunks[i].Images {
chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL)
chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption)
chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText)
chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL)
}
}
for i := range options.ParentChunks {
options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content)
}
// Check if knowledge is being deleted/cancelled before processing.
// Both statuses short-circuit identically here — there's nothing to clean
// up yet so the branch is purely "stop early".
@ -2896,6 +2914,7 @@ func (s *knowledgeService) UpdateImageInfo(
chunkID string,
imageInfo string,
) error {
imageInfo = common.CleanInvalidUTF8(imageInfo)
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(imageInfo), &images); err != nil {
logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)
@ -3516,6 +3535,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// Step 3: Split into chunks using Go chunker. Browser textareas normalize
// pasted content to LF, so normalize uploaded source text before calculating
// chunk boundaries as well.
sanitizeReadResult(convertResult)
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
@ -3572,6 +3592,28 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
// sanitizeReadResult protects every text field that can cross from a parser
// into the embedding, storage, or tracing layers. A parser may return a Go
// string containing arbitrary bytes even though the string type itself does
// not enforce UTF-8 validity.
func sanitizeReadResult(result *types.ReadResult) {
if result == nil {
return
}
result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent)
result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath)
result.Error = common.CleanInvalidUTF8(result.Error)
for key, value := range result.Metadata {
result.Metadata[key] = common.CleanInvalidUTF8(value)
}
for i := range result.ImageRefs {
result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename)
result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef)
result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType)
result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey)
}
}
// convert handles both file and URL reading using a unified ReadRequest.
func (s *knowledgeService) convert(
ctx context.Context,
@ -3689,6 +3731,7 @@ func (s *knowledgeService) convert(
code, "document read failed", err)
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
}
sanitizeReadResult(result)
if result.Error != "" {
logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s",
kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error)

View File

@ -1,6 +1,9 @@
package common
import "testing"
import (
"testing"
"unicode/utf8"
)
func TestParseLLMJsonResponse(t *testing.T) {
type payload struct {
@ -39,6 +42,18 @@ func TestParseLLMJsonResponse(t *testing.T) {
}
}
func TestCleanInvalidUTF8(t *testing.T) {
input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail"
got := CleanInvalidUTF8(input)
if !utf8.ValidString(got) {
t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got))
}
if want := "valid .tail"; got != want {
t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want)
}
}
func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) {
const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```"
type payload struct {