fix(chunk): sanitize context headers before persistence

main
shangfeng-liu 2026-08-14 00:33:09 +08:00
parent 49010d2526
commit b6b1148bbb
2 changed files with 21 additions and 0 deletions

View File

@ -42,6 +42,7 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
chunk.ContextHeader = common.CleanInvalidUTF8(chunk.ContextHeader)
if chunk.SourceContent == "" {
chunk.SourceContent = chunk.Content
}

View File

@ -3,6 +3,7 @@ package repository
import (
"context"
"testing"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
@ -63,6 +64,25 @@ func TestCreateChunks_SQLite_SeqIDAutoAssigned(t *testing.T) {
}
}
func TestCreateChunks_CleansContextHeaderBeforePersistence(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
chunk := makeChunk(uuid.New().String(), uuid.New().String(), types.ChunkTypeText)
chunk.ContextHeader = "# \x00投资评级说明\xff"
require.NoError(t, repo.CreateChunks(context.Background(), []*types.Chunk{chunk}))
require.Equal(t, "# 投资评级说明", chunk.ContextHeader)
require.NotContains(t, chunk.ContextHeader, "\x00")
require.True(t, utf8.ValidString(chunk.ContextHeader))
var saved types.Chunk
require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)
assert.Equal(t, "# 投资评级说明", saved.ContextHeader)
assert.NotContains(t, saved.ContextHeader, "\x00")
assert.True(t, utf8.ValidString(saved.ContextHeader))
}
func TestCreateChunks_SQLite_SeqIDContinuesFromExisting(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)