diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 4f0aecbf..17be8e55 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -42,6 +42,7 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository { func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { for _, chunk := range chunks { chunk.Content = common.CleanInvalidUTF8(chunk.Content) + chunk.ContextHeader = common.CleanInvalidUTF8(chunk.ContextHeader) if chunk.SourceContent == "" { chunk.SourceContent = chunk.Content } diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.go index 34163c81..10465272 100644 --- a/internal/application/repository/chunk_sqlite_test.go +++ b/internal/application/repository/chunk_sqlite_test.go @@ -3,6 +3,7 @@ package repository import ( "context" "testing" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" @@ -63,6 +64,25 @@ func TestCreateChunks_SQLite_SeqIDAutoAssigned(t *testing.T) { } } +func TestCreateChunks_CleansContextHeaderBeforePersistence(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + chunk := makeChunk(uuid.New().String(), uuid.New().String(), types.ChunkTypeText) + chunk.ContextHeader = "# \x00投资评级说明\xff" + + require.NoError(t, repo.CreateChunks(context.Background(), []*types.Chunk{chunk})) + + require.Equal(t, "# 投资评级说明", chunk.ContextHeader) + require.NotContains(t, chunk.ContextHeader, "\x00") + require.True(t, utf8.ValidString(chunk.ContextHeader)) + + var saved types.Chunk + require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error) + assert.Equal(t, "# 投资评级说明", saved.ContextHeader) + assert.NotContains(t, saved.ContextHeader, "\x00") + assert.True(t, utf8.ValidString(saved.ContextHeader)) +} + func TestCreateChunks_SQLite_SeqIDContinuesFromExisting(t *testing.T) { db := setupChunkTestDB(t) repo := NewChunkRepository(db)