From 801005f736b5e76c514588f3ac9abe5212ebb219 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:43:42 -0400 Subject: [PATCH] fix(knowledge): keep long non-ASCII folder names valid UTF-8 NormalizeKnowledgeFolderPath capped a directory name with segment[:MaxKnowledgeFolderSegmentLength] (internal/types/knowledge_folder.go:51). That is a raw byte index, so a name longer than 128 bytes was cut wherever byte 128 landed. For a Chinese folder name that is inside the 43rd character, and the leading bytes of the half-cut character stay in the string. The result is an invalid UTF-8 folder_path. json.Marshal replaces the broken bytes with U+FFFD, so the path the client reads back is not the path stored in the row, and RenameKnowledgeFolder (internal/application/service/knowledge.go:611) normalizes that returned value and matches nothing. The cut now backs off to the nearest rune start. The cap stays a byte budget, so a 60-character Chinese name keeps its first 42 characters (126 bytes) instead of 128 bytes ending in half a character. ASCII names are unchanged. --- internal/types/knowledge_folder.go | 11 ++++++++++- internal/types/knowledge_folder_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/types/knowledge_folder.go b/internal/types/knowledge_folder.go index 3d314db7..c1968917 100644 --- a/internal/types/knowledge_folder.go +++ b/internal/types/knowledge_folder.go @@ -3,6 +3,7 @@ package types import ( "sort" "strings" + "unicode/utf8" ) const ( @@ -49,7 +50,15 @@ func NormalizeKnowledgeFolderPath(raw string) string { continue } if len(segment) > MaxKnowledgeFolderSegmentLength { - segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength]) + // The cap is a byte budget, so back the cut off to the start of a + // rune. Slicing at the raw index leaves the leading bytes of a + // partial rune behind, and a folder named in Chinese hits that on + // the 43rd character. + cut := MaxKnowledgeFolderSegmentLength + for cut > 0 && !utf8.RuneStart(segment[cut]) { + cut-- + } + segment = strings.TrimSpace(segment[:cut]) } if segment == "" { continue diff --git a/internal/types/knowledge_folder_test.go b/internal/types/knowledge_folder_test.go index f45ee62a..55275750 100644 --- a/internal/types/knowledge_folder_test.go +++ b/internal/types/knowledge_folder_test.go @@ -3,6 +3,7 @@ package types import ( "strings" "testing" + "unicode/utf8" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -44,6 +45,30 @@ func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) { assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength) } +func TestNormalizeKnowledgeFolderPathKeepsSegmentsValidUTF8(t *testing.T) { + // 60 Chinese characters is 180 bytes, so the segment cap cuts inside the + // 43rd character. The stored folder_path has to stay valid UTF-8: an + // invalid one no longer survives a JSON round trip, so the path the client + // gets back never matches the row again on rename or move. + segment := strings.Repeat("中", 60) + normalized := NormalizeKnowledgeFolderPath(segment) + + assert.True(t, utf8.ValidString(normalized), + "segment truncated mid-rune: % x", []byte(normalized)) + assert.LessOrEqual(t, len(normalized), MaxKnowledgeFolderSegmentLength, + "the cap is a byte budget and must still hold") + assert.True(t, strings.HasPrefix(segment, normalized), + "truncation must only drop a suffix") + assert.Equal(t, normalized, NormalizeKnowledgeFolderPath(normalized), + "normalizing an already-normalized path must be a no-op") + + // A multi-byte character straddling the cap in an otherwise ASCII name. + mixed := strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1) + "é" + "tail" + assert.True(t, utf8.ValidString(NormalizeKnowledgeFolderPath(mixed))) + assert.Equal(t, strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1), + NormalizeKnowledgeFolderPath(mixed)) +} + func TestSplitKnowledgeRelativePath(t *testing.T) { cases := []struct { in string