fix(knowledge): keep long non-ASCII folder names valid UTF-8
NormalizeKnowledgeFolderPath capped a directory name with segment[:MaxKnowledgeFolderSegmentLength] (internal/types/knowledge_folder.go:51). That is a raw byte index, so a name longer than 128 bytes was cut wherever byte 128 landed. For a Chinese folder name that is inside the 43rd character, and the leading bytes of the half-cut character stay in the string. The result is an invalid UTF-8 folder_path. json.Marshal replaces the broken bytes with U+FFFD, so the path the client reads back is not the path stored in the row, and RenameKnowledgeFolder (internal/application/service/knowledge.go:611) normalizes that returned value and matches nothing. The cut now backs off to the nearest rune start. The cap stays a byte budget, so a 60-character Chinese name keeps its first 42 characters (126 bytes) instead of 128 bytes ending in half a character. ASCII names are unchanged.main
parent
4fb0b08719
commit
801005f736
|
|
@ -3,6 +3,7 @@ package types
|
|||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -49,7 +50,15 @@ func NormalizeKnowledgeFolderPath(raw string) string {
|
|||
continue
|
||||
}
|
||||
if len(segment) > MaxKnowledgeFolderSegmentLength {
|
||||
segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength])
|
||||
// The cap is a byte budget, so back the cut off to the start of a
|
||||
// rune. Slicing at the raw index leaves the leading bytes of a
|
||||
// partial rune behind, and a folder named in Chinese hits that on
|
||||
// the 43rd character.
|
||||
cut := MaxKnowledgeFolderSegmentLength
|
||||
for cut > 0 && !utf8.RuneStart(segment[cut]) {
|
||||
cut--
|
||||
}
|
||||
segment = strings.TrimSpace(segment[:cut])
|
||||
}
|
||||
if segment == "" {
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package types
|
|||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -44,6 +45,30 @@ func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) {
|
|||
assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength)
|
||||
}
|
||||
|
||||
func TestNormalizeKnowledgeFolderPathKeepsSegmentsValidUTF8(t *testing.T) {
|
||||
// 60 Chinese characters is 180 bytes, so the segment cap cuts inside the
|
||||
// 43rd character. The stored folder_path has to stay valid UTF-8: an
|
||||
// invalid one no longer survives a JSON round trip, so the path the client
|
||||
// gets back never matches the row again on rename or move.
|
||||
segment := strings.Repeat("中", 60)
|
||||
normalized := NormalizeKnowledgeFolderPath(segment)
|
||||
|
||||
assert.True(t, utf8.ValidString(normalized),
|
||||
"segment truncated mid-rune: % x", []byte(normalized))
|
||||
assert.LessOrEqual(t, len(normalized), MaxKnowledgeFolderSegmentLength,
|
||||
"the cap is a byte budget and must still hold")
|
||||
assert.True(t, strings.HasPrefix(segment, normalized),
|
||||
"truncation must only drop a suffix")
|
||||
assert.Equal(t, normalized, NormalizeKnowledgeFolderPath(normalized),
|
||||
"normalizing an already-normalized path must be a no-op")
|
||||
|
||||
// A multi-byte character straddling the cap in an otherwise ASCII name.
|
||||
mixed := strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1) + "é" + "tail"
|
||||
assert.True(t, utf8.ValidString(NormalizeKnowledgeFolderPath(mixed)))
|
||||
assert.Equal(t, strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1),
|
||||
NormalizeKnowledgeFolderPath(mixed))
|
||||
}
|
||||
|
||||
func TestSplitKnowledgeRelativePath(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
|
|
|
|||
Loading…
Reference in New Issue