fix(chunk): sanitize context headers before persistence
parent
49010d2526
commit
b6b1148bbb
|
|
@ -42,6 +42,7 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
|
|||
func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
|
||||
for _, chunk := range chunks {
|
||||
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
|
||||
chunk.ContextHeader = common.CleanInvalidUTF8(chunk.ContextHeader)
|
||||
if chunk.SourceContent == "" {
|
||||
chunk.SourceContent = chunk.Content
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package repository
|
|||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
|
|
@ -63,6 +64,25 @@ func TestCreateChunks_SQLite_SeqIDAutoAssigned(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCreateChunks_CleansContextHeaderBeforePersistence(t *testing.T) {
|
||||
db := setupChunkTestDB(t)
|
||||
repo := NewChunkRepository(db)
|
||||
chunk := makeChunk(uuid.New().String(), uuid.New().String(), types.ChunkTypeText)
|
||||
chunk.ContextHeader = "# \x00投资评级说明\xff"
|
||||
|
||||
require.NoError(t, repo.CreateChunks(context.Background(), []*types.Chunk{chunk}))
|
||||
|
||||
require.Equal(t, "# 投资评级说明", chunk.ContextHeader)
|
||||
require.NotContains(t, chunk.ContextHeader, "\x00")
|
||||
require.True(t, utf8.ValidString(chunk.ContextHeader))
|
||||
|
||||
var saved types.Chunk
|
||||
require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)
|
||||
assert.Equal(t, "# 投资评级说明", saved.ContextHeader)
|
||||
assert.NotContains(t, saved.ContextHeader, "\x00")
|
||||
assert.True(t, utf8.ValidString(saved.ContextHeader))
|
||||
}
|
||||
|
||||
func TestCreateChunks_SQLite_SeqIDContinuesFromExisting(t *testing.T) {
|
||||
db := setupChunkTestDB(t)
|
||||
repo := NewChunkRepository(db)
|
||||
|
|
|
|||
Loading…
Reference in New Issue