fix(knowledge): keep long non-ASCII folder names valid UTF-8

NormalizeKnowledgeFolderPath capped a directory name with
segment[:MaxKnowledgeFolderSegmentLength] (internal/types/knowledge_folder.go:51).
That is a raw byte index, so a name longer than 128 bytes was cut wherever
byte 128 landed. For a Chinese folder name that is inside the 43rd character,
and the leading bytes of the half-cut character stay in the string.

The result is an invalid UTF-8 folder_path. json.Marshal replaces the broken
bytes with U+FFFD, so the path the client reads back is not the path stored in
the row, and RenameKnowledgeFolder (internal/application/service/knowledge.go:611)
normalizes that returned value and matches nothing.

The cut now backs off to the nearest rune start. The cap stays a byte budget,
so a 60-character Chinese name keeps its first 42 characters (126 bytes)
instead of 128 bytes ending in half a character. ASCII names are unchanged.
main
Max Freedom Pollard 2026-09-06 12:43:42 -04:00 committed by lyingbug
parent 4fb0b08719
commit 801005f736
2 changed files with 35 additions and 1 deletions

View File

@ -3,6 +3,7 @@ package types
import (
"sort"
"strings"
"unicode/utf8"
)
const (
@ -49,7 +50,15 @@ func NormalizeKnowledgeFolderPath(raw string) string {
continue
}
if len(segment) > MaxKnowledgeFolderSegmentLength {
segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength])
// The cap is a byte budget, so back the cut off to the start of a
// rune. Slicing at the raw index leaves the leading bytes of a
// partial rune behind, and a folder named in Chinese hits that on
// the 43rd character.
cut := MaxKnowledgeFolderSegmentLength
for cut > 0 && !utf8.RuneStart(segment[cut]) {
cut--
}
segment = strings.TrimSpace(segment[:cut])
}
if segment == "" {
continue

View File

@ -3,6 +3,7 @@ package types
import (
"strings"
"testing"
"unicode/utf8"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -44,6 +45,30 @@ func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) {
assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength)
}
func TestNormalizeKnowledgeFolderPathKeepsSegmentsValidUTF8(t *testing.T) {
// 60 Chinese characters is 180 bytes, so the segment cap cuts inside the
// 43rd character. The stored folder_path has to stay valid UTF-8: an
// invalid one no longer survives a JSON round trip, so the path the client
// gets back never matches the row again on rename or move.
segment := strings.Repeat("中", 60)
normalized := NormalizeKnowledgeFolderPath(segment)
assert.True(t, utf8.ValidString(normalized),
"segment truncated mid-rune: % x", []byte(normalized))
assert.LessOrEqual(t, len(normalized), MaxKnowledgeFolderSegmentLength,
"the cap is a byte budget and must still hold")
assert.True(t, strings.HasPrefix(segment, normalized),
"truncation must only drop a suffix")
assert.Equal(t, normalized, NormalizeKnowledgeFolderPath(normalized),
"normalizing an already-normalized path must be a no-op")
// A multi-byte character straddling the cap in an otherwise ASCII name.
mixed := strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1) + "é" + "tail"
assert.True(t, utf8.ValidString(NormalizeKnowledgeFolderPath(mixed)))
assert.Equal(t, strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1),
NormalizeKnowledgeFolderPath(mixed))
}
func TestSplitKnowledgeRelativePath(t *testing.T) {
cases := []struct {
in string