fix(markdown): sanitize invalid UTF-8 and NUL bytes
parent
26d9552642
commit
786d67609d
|
|
@ -6,6 +6,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"gorm.io/gorm"
|
||||
|
|
@ -53,6 +54,7 @@ func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository {
|
|||
|
||||
// CreateKnowledge creates knowledge
|
||||
func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
|
||||
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
|
||||
err := r.db.WithContext(ctx).Create(knowledge).Error
|
||||
return err
|
||||
}
|
||||
|
|
@ -306,6 +308,7 @@ func (r *knowledgeRepository) RenameKnowledgeFolderPath(
|
|||
|
||||
// UpdateKnowledge updates knowledge
|
||||
func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
|
||||
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
|
||||
omit := omitFieldsOnUpdate
|
||||
// Legacy/unit-test schemas created before custom_metadata should continue
|
||||
// to support unrelated updates when the caller did not provide the field.
|
||||
|
|
@ -321,6 +324,11 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledg
|
|||
if len(knowledgeList) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, knowledge := range knowledgeList {
|
||||
if knowledge != nil {
|
||||
knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
|
||||
}
|
||||
}
|
||||
return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error
|
||||
}
|
||||
|
||||
|
|
@ -513,6 +521,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(
|
|||
column string,
|
||||
value interface{},
|
||||
) error {
|
||||
if column == "error_message" {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
value = common.CleanInvalidUTF8(v)
|
||||
case []byte:
|
||||
value = common.CleanInvalidUTF8(string(v))
|
||||
}
|
||||
}
|
||||
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error
|
||||
return err
|
||||
}
|
||||
|
|
@ -529,6 +545,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(
|
|||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
if value, ok := values["error_message"]; ok {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
values["error_message"] = common.CleanInvalidUTF8(v)
|
||||
case []byte:
|
||||
values["error_message"] = common.CleanInvalidUTF8(string(v))
|
||||
}
|
||||
}
|
||||
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
|
|
@ -99,6 +100,23 @@ func reloadKnowledgeErrorMessage(t *testing.T, db *gorm.DB, id string) string {
|
|||
return msg
|
||||
}
|
||||
|
||||
func TestKnowledgeRepository_UpdateKnowledgeColumnsSanitizesErrorMessage(t *testing.T) {
|
||||
db := setupKnowledgeTestDB(t)
|
||||
repo := NewKnowledgeRepository(db)
|
||||
id := insertProcessingKnowledge(t, db)
|
||||
invalid := "parse failed " + string([]byte{0xef, 0xbc, 0x2e})
|
||||
|
||||
require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), id, map[string]interface{}{
|
||||
"error_message": invalid,
|
||||
}))
|
||||
|
||||
got := reloadKnowledgeErrorMessage(t, db, id)
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("persisted error_message is invalid UTF-8: % x", []byte(got))
|
||||
}
|
||||
assert.Equal(t, "parse failed .", got)
|
||||
}
|
||||
|
||||
func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {
|
||||
t.Helper()
|
||||
id := uuid.New().String()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
|
@ -61,6 +62,12 @@ func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.Knowled
|
|||
if row.Attempt == 0 {
|
||||
row.Attempt = 1
|
||||
}
|
||||
// Error fields may originate from third-party parser/model responses.
|
||||
// Normalize them at the persistence boundary so SQLite/PostgreSQL and the
|
||||
// tracing API never receive malformed UTF-8.
|
||||
row.ErrorCode = common.CleanInvalidUTF8(row.ErrorCode)
|
||||
row.ErrorMessage = common.CleanInvalidUTF8(row.ErrorMessage)
|
||||
row.ErrorDetail = common.CleanInvalidUTF8(row.ErrorDetail)
|
||||
// We let GORM populate created_at/updated_at via the autoCreate /
|
||||
// autoUpdate tags. ON CONFLICT updates only the fields that may
|
||||
// transition between calls — name/kind/parent are immutable once
|
||||
|
|
@ -163,6 +170,7 @@ func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID strin
|
|||
// Postgres-specific WITH RECURSIVE would be denser but harder to test on
|
||||
// the SQLite Lite backend. The iterative path stays portable.
|
||||
func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) {
|
||||
reason = common.CleanInvalidUTF8(reason)
|
||||
frontier := []string{parentSpanID}
|
||||
var totalAffected int64
|
||||
for depth := 0; depth < 16 && len(frontier) > 0; depth++ {
|
||||
|
|
@ -215,6 +223,8 @@ func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowled
|
|||
func (r *knowledgeSpanRepository) CancelAllOpenSpans(
|
||||
ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,
|
||||
) (int64, error) {
|
||||
errorCode = common.CleanInvalidUTF8(errorCode)
|
||||
reason = common.CleanInvalidUTF8(reason)
|
||||
now := time.Now()
|
||||
updates := map[string]any{
|
||||
"status": types.SpanStatusCancelled,
|
||||
|
|
@ -240,6 +250,8 @@ func (r *knowledgeSpanRepository) CancelOpenSpansByName(
|
|||
if knowledgeID == "" || attempt <= 0 || name == "" {
|
||||
return 0, nil
|
||||
}
|
||||
errorCode = common.CleanInvalidUTF8(errorCode)
|
||||
reason = common.CleanInvalidUTF8(reason)
|
||||
now := time.Now()
|
||||
res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
|
||||
Where("knowledge_id = ? AND attempt = ? AND name = ? AND status IN ?",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
|
@ -85,6 +86,37 @@ func TestKnowledgeSpanRepo_UpsertAndList(t *testing.T) {
|
|||
assert.Equal(t, int64(2000), rows[0].DurationMs)
|
||||
}
|
||||
|
||||
func TestKnowledgeSpanRepo_UpsertSanitizesErrorFields(t *testing.T) {
|
||||
repo, _ := setupSpanTestRepo(t)
|
||||
now := time.Now()
|
||||
invalid := "prefix " + string([]byte{0xef, 0xbc, 0x2e}) + " suffix"
|
||||
|
||||
require.NoError(t, repo.Upsert(context.Background(), &types.KnowledgeProcessingSpan{
|
||||
KnowledgeID: "kid-invalid-utf8",
|
||||
Attempt: 1,
|
||||
SpanID: "span-invalid",
|
||||
Name: types.StageDocReader,
|
||||
Kind: types.SpanKindStage,
|
||||
Status: types.SpanStatusFailed,
|
||||
ErrorCode: invalid,
|
||||
ErrorMessage: invalid,
|
||||
ErrorDetail: invalid,
|
||||
StartedAt: &now,
|
||||
}))
|
||||
|
||||
rows, err := repo.ListByAttempt(context.Background(), "kid-invalid-utf8", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
for _, value := range []string{rows[0].ErrorCode, rows[0].ErrorMessage, rows[0].ErrorDetail} {
|
||||
if !utf8.ValidString(value) {
|
||||
t.Fatalf("persisted error field is invalid UTF-8: % x", []byte(value))
|
||||
}
|
||||
if value != "prefix . suffix" {
|
||||
t.Fatalf("persisted error field = %q, want %q", value, "prefix . suffix")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestKnowledgeSpanRepo_NextAttempt confirms that NextAttempt allocates
|
||||
// a fresh number per knowledge, isolating reparse history. Critical
|
||||
// because the API layer renders attempt history by this number.
|
||||
|
|
@ -182,6 +214,45 @@ func TestKnowledgeSpanRepo_CancelOpenSpansByName(t *testing.T) {
|
|||
assert.Equal(t, types.SpanStatusRunning, statusBy["q-old"])
|
||||
}
|
||||
|
||||
func TestKnowledgeSpanRepo_CancelPathsSanitizeErrorFields(t *testing.T) {
|
||||
repo, _ := setupSpanTestRepo(t)
|
||||
ctx := context.Background()
|
||||
kid := "kid-cancel-invalid-utf8"
|
||||
now := time.Now()
|
||||
for _, r := range []*types.KnowledgeProcessingSpan{
|
||||
{KnowledgeID: kid, Attempt: 1, SpanID: "root", Name: "root", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now},
|
||||
{KnowledgeID: kid, Attempt: 1, SpanID: "descendant", ParentSpanID: "root", Name: "descendant", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
|
||||
{KnowledgeID: kid, Attempt: 1, SpanID: "bulk", Name: "bulk", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
|
||||
{KnowledgeID: kid, Attempt: 1, SpanID: "named", Name: "named", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
|
||||
} {
|
||||
require.NoError(t, repo.Upsert(ctx, r))
|
||||
}
|
||||
|
||||
invalidCode := "CODE " + string([]byte{0xef, 0xbc, 0x2e})
|
||||
invalidReason := "reason " + string([]byte{0xef, 0xbc, 0x2e})
|
||||
_, err := repo.CancelOpenSpansByName(ctx, kid, 1, "named", invalidCode, invalidReason)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CancelDescendants(ctx, kid, 1, "root", invalidReason)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CancelAllOpenSpans(ctx, kid, 1, invalidCode, invalidReason)
|
||||
require.NoError(t, err)
|
||||
|
||||
rows, err := repo.ListByAttempt(ctx, kid, 1)
|
||||
require.NoError(t, err)
|
||||
byID := make(map[string]types.KnowledgeProcessingSpan, len(rows))
|
||||
for _, row := range rows {
|
||||
byID[row.SpanID] = row
|
||||
}
|
||||
assert.Equal(t, "UPSTREAM_FAILED", byID["descendant"].ErrorCode)
|
||||
assert.Equal(t, "reason .", byID["descendant"].ErrorMessage)
|
||||
for _, id := range []string{"root", "bulk", "named"} {
|
||||
assert.Equal(t, "CODE .", byID[id].ErrorCode, id)
|
||||
assert.Equal(t, "reason .", byID[id].ErrorMessage, id)
|
||||
assert.True(t, utf8.ValidString(byID[id].ErrorCode), id)
|
||||
assert.True(t, utf8.ValidString(byID[id].ErrorMessage), id)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKnowledgeSpanRepo_ListAttemptIsolation guarantees that different
|
||||
// attempts of the same knowledge stay queryable independently — the
|
||||
// foundation for the "show history" UI navigation (?attempt=N).
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
|
|
@ -251,6 +252,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
|||
options = opts[0]
|
||||
}
|
||||
|
||||
// Parser output and manually supplied passages can contain malformed byte
|
||||
// sequences. Clean them before logging, chunk persistence, or embedding;
|
||||
// the embedding provider and tracing/database drivers expect valid UTF-8.
|
||||
for i := range chunks {
|
||||
chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content)
|
||||
chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader)
|
||||
for j := range chunks[i].Images {
|
||||
chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL)
|
||||
chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption)
|
||||
chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText)
|
||||
chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL)
|
||||
}
|
||||
}
|
||||
for i := range options.ParentChunks {
|
||||
options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content)
|
||||
}
|
||||
|
||||
// Check if knowledge is being deleted/cancelled before processing.
|
||||
// Both statuses short-circuit identically here — there's nothing to clean
|
||||
// up yet so the branch is purely "stop early".
|
||||
|
|
@ -2896,6 +2914,7 @@ func (s *knowledgeService) UpdateImageInfo(
|
|||
chunkID string,
|
||||
imageInfo string,
|
||||
) error {
|
||||
imageInfo = common.CleanInvalidUTF8(imageInfo)
|
||||
var images []*types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(imageInfo), &images); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)
|
||||
|
|
@ -3516,6 +3535,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
|||
// Step 3: Split into chunks using Go chunker. Browser textareas normalize
|
||||
// pasted content to LF, so normalize uploaded source text before calculating
|
||||
// chunk boundaries as well.
|
||||
sanitizeReadResult(convertResult)
|
||||
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
|
||||
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
|
||||
|
||||
|
|
@ -3572,6 +3592,28 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
|||
return nil
|
||||
}
|
||||
|
||||
// sanitizeReadResult protects every text field that can cross from a parser
|
||||
// into the embedding, storage, or tracing layers. A parser may return a Go
|
||||
// string containing arbitrary bytes even though the string type itself does
|
||||
// not enforce UTF-8 validity.
|
||||
func sanitizeReadResult(result *types.ReadResult) {
|
||||
if result == nil {
|
||||
return
|
||||
}
|
||||
result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent)
|
||||
result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath)
|
||||
result.Error = common.CleanInvalidUTF8(result.Error)
|
||||
for key, value := range result.Metadata {
|
||||
result.Metadata[key] = common.CleanInvalidUTF8(value)
|
||||
}
|
||||
for i := range result.ImageRefs {
|
||||
result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename)
|
||||
result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef)
|
||||
result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType)
|
||||
result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey)
|
||||
}
|
||||
}
|
||||
|
||||
// convert handles both file and URL reading using a unified ReadRequest.
|
||||
func (s *knowledgeService) convert(
|
||||
ctx context.Context,
|
||||
|
|
@ -3689,6 +3731,7 @@ func (s *knowledgeService) convert(
|
|||
code, "document read failed", err)
|
||||
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
|
||||
}
|
||||
sanitizeReadResult(result)
|
||||
if result.Error != "" {
|
||||
logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s",
|
||||
kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package common
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestParseLLMJsonResponse(t *testing.T) {
|
||||
type payload struct {
|
||||
|
|
@ -39,6 +42,18 @@ func TestParseLLMJsonResponse(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCleanInvalidUTF8(t *testing.T) {
|
||||
input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail"
|
||||
|
||||
got := CleanInvalidUTF8(input)
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got))
|
||||
}
|
||||
if want := "valid .tail"; got != want {
|
||||
t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) {
|
||||
const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```"
|
||||
type payload struct {
|
||||
|
|
|
|||
Loading…
Reference in New Issue