From 820a14dafa0ef9c6405baffebc70a3b631bfb95d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 17 Sep 2026 15:59:15 +0800 Subject: [PATCH] refactor(datasource): share connector filename sanitization --- docs/code-slimming-audit.md | 14 ++++++- .../datasource/connector/ima/connector.go | 30 +------------ .../connector/ima/connector_test.go | 32 ++++++++++++++ .../datasource/connector/ima/types_test.go | 19 --------- .../connector/rss/connector_test.go | 25 +++++++++++ internal/datasource/connector/rss/types.go | 31 ++------------ .../datasource/connector/yuque/connector.go | 2 +- .../connector/yuque/connector_test.go | 29 +++++++++++++ internal/datasource/connector/yuque/types.go | 32 -------------- .../datasource/connector/yuque/types_test.go | 19 --------- internal/datasource/filename.go | 34 +++++++++++++++ internal/datasource/filename_test.go | 42 +++++++++++++++++++ 12 files changed, 180 insertions(+), 129 deletions(-) create mode 100644 internal/datasource/filename.go create mode 100644 internal/datasource/filename_test.go diff --git a/docs/code-slimming-audit.md b/docs/code-slimming-audit.md index b560d3da..abfa34d7 100644 --- a/docs/code-slimming-audit.md +++ b/docs/code-slimming-audit.md @@ -98,6 +98,16 @@ - 本批只删除无调用代码,复用现有包测试和编译检查,没有添加仅断言函数不存在的测试。 - 推送检查暴露新合入的 fork 测试未隔离 Git hook 环境:继承的 `GIT_DIR` 等变量使临时仓库操作指向当前仓库。测试辅助函数现清除继承的 `GIT_*` 变量;真实仓库测试主动注入指向临时路径的仓库变量,验证旧实现失败、修复后通过。该测试修复单独提交,不改变生产逻辑。 +## 第六批已实施:外部来源文件名清理 + +基于第五批合入后的 `10973ed32`,IMA、语雀、RSS 复用 `datasource.SanitizeFileName`,统一无效标点替换、空标题回退和 200 字节 UTF-8 边界截断。计入新增共享实现后,生产 Go 源码净减少 49 行。 + +- IMA 的媒体与笔记路径、语雀文档直接调用共享函数;RSS 仅保留换行/制表符转空格和首尾空白清理,再调用共享函数。 +- 保留原有扩展名拼接、源标题、空白、控制字符和无效 UTF-8 输入处理行为。飞书的扩展名保留、钉钉的控制字符/尾部标点清理、Confluence 的按字符截断规则不同,本批未合并。 +- 将两份旧 helper 测试收敛到共享函数测试,补充 16 个边界用例;在实际连接器读取/组装路径新增 15 个用例,覆盖 IMA 两类内容、语雀和 RSS 的最终文件名。 +- 用重构前的三个实现对 30,000 组确定性生成输入执行差异校验,结果一致;输入包含中文、emoji、空白、非法字节和截断边界。临时对比代码未留在仓库,避免长期维护旧实现副本。 +- 4 个受影响包测试通过:104 项顶层测试,包含既有同步、游标和错误处理测试。未修改前端,未运行浏览器检查或真实外部来源服务测试。 + ## 后续优先级 | 优先级 | 证据 | 建议边界 | 验证重点 | @@ -113,7 +123,7 @@ ### 后端重复的具体判断 - Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。第五批已删除 Weaviate 的失效函数,保留 Qdrant 实现。 -- IMA、语雀的 `sanitizeFileName` 函数体相同,可以共享;RSS 的近似实现另外处理首尾空白和换行,合并时必须保留这些差异。 +- 第六批已合并 IMA、语雀、RSS 的文件名标点替换和 UTF-8 截断逻辑;RSS 的空白预处理保留在连接器内。飞书、钉钉、Confluence 的其他规则继续独立维护。 - Milvus/Qdrant/Weaviate 的结果映射体相似,但接收不同后端类型。当前各约 17 行,为此引入通用接口或泛型转换层未必减少维护成本。 - 终端和桌面 WebSocket ticket handler 重复检查会话、用户、租户和 token,但签发方式不同。后续可共享前置身份校验,保留 ticket 和连接生命周期各自的契约。 - 资源权限已经有 `internal/application/access`;继续沿已有授权边界收敛,避免新建与其并行的权限工具层。 @@ -152,4 +162,4 @@ - 构建仍报告大 chunk 提示;本轮未以 bundle 体积为目标,也没有测量构建前后的体积差异。 - 未修改后端,未运行后端测试;未做浏览器视觉验收。 -下一批优先处理 IMA、语雀等外部来源的重复文件名清理逻辑,核实调用与差异后再决定共享边界。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。 +下一批优先检查终端与桌面 WebSocket ticket 的重复身份校验,沿现有授权边界确定共享范围。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。 diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go index 89c305a1..6111ffc6 100644 --- a/internal/datasource/connector/ima/connector.go +++ b/internal/datasource/connector/ima/connector.go @@ -7,7 +7,6 @@ import ( "sort" "strings" "time" - "unicode/utf8" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" @@ -437,7 +436,7 @@ func fetchNote( return types.FetchedItem{}, fetchSkipped } - fileName := sanitizeFileName(f.Title) + fileName := datasource.SanitizeFileName(f.Title) if !strings.HasSuffix(strings.ToLower(fileName), ".md") { fileName += ".md" } @@ -540,7 +539,7 @@ func fetchOneMedia( ct = mimeForExtension(ext) } - fileName := sanitizeFileName(f.Title) + fileName := datasource.SanitizeFileName(f.Title) if !strings.HasSuffix(strings.ToLower(fileName), "."+ext) { fileName = fileName + "." + ext } @@ -585,28 +584,3 @@ func baseMetadata( } return m } - -// sanitizeFileName removes filesystem-hostile characters and truncates to a -// safe UTF-8 boundary. -func sanitizeFileName(name string) string { - if name == "" { - return "untitled" - } - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - ) - result := replacer.Replace(name) - const maxBytes = 200 - if len(result) > maxBytes { - result = result[:maxBytes] - for len(result) > 0 { - r, size := utf8.DecodeLastRuneInString(result) - if r != utf8.RuneError || size != 1 { - break - } - result = result[:len(result)-1] - } - } - return result -} diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go index d1a447e7..42970c8c 100644 --- a/internal/datasource/connector/ima/connector_test.go +++ b/internal/datasource/connector/ima/connector_test.go @@ -509,3 +509,35 @@ func TestConnectorIsRegisteredInMetadata(t *testing.T) { t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA) } } + +func TestFetchAll_FileNamePolicy(t *testing.T) { + for _, media := range []struct { + name string + kind int32 + }{{"media", mediaTypeMarkdown}, {"note", mediaTypeNote}} { + for _, tt := range []struct{ name, title, want string }{ + {"punctuation", "A/B:C", "A_B_C.md"}, + {"existing extension and whitespace", " \tA/B.MD", " \tA_B.MD"}, + {"long title", strings.Repeat("测", 100) + ".MD", strings.Repeat("测", 66) + ".md"}, + } { + t.Run(media.name+"/"+tt.name, func(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{ + MediaID: "file", Title: tt.title, MediaType: media.kind, + Body: "# content", NotebookID: "123", NoteBody: "# content", + }}) + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 { + t.Fatalf("got %d items, want 1", len(items)) + } + if items[0].Title != tt.title || items[0].FileName != tt.want { + t.Fatalf("title = %q, filename = %q; want %q, %q", + items[0].Title, items[0].FileName, tt.title, tt.want) + } + }) + } + } +} diff --git a/internal/datasource/connector/ima/types_test.go b/internal/datasource/connector/ima/types_test.go index d7859c04..e52f6ff6 100644 --- a/internal/datasource/connector/ima/types_test.go +++ b/internal/datasource/connector/ima/types_test.go @@ -131,25 +131,6 @@ func TestExtensionForContentType(t *testing.T) { } } -func TestSanitizeFileName(t *testing.T) { - if got := sanitizeFileName(""); got != "untitled" { - t.Errorf("empty name = %q, want untitled", got) - } - if got := sanitizeFileName(`a/b\c:d*e?f"gi|j`); strings.ContainsAny(got, `/\:*?"<>|`) { - t.Errorf("filesystem-hostile characters survived: %q", got) - } - - // A long CJK title must be truncated on a rune boundary, not mid-sequence. - long := strings.Repeat("知识", 200) - got := sanitizeFileName(long) - if len(got) > 200 { - t.Errorf("len = %d bytes, want <= 200", len(got)) - } - if !strings.HasPrefix(long, got) { - t.Errorf("truncation corrupted the prefix: %q", got) - } -} - func TestIsSkippableMediaType(t *testing.T) { for _, mt := range []int32{mediaTypeAISession, mediaTypeVideo} { if !isSkippableMediaType(mt) { diff --git a/internal/datasource/connector/rss/connector_test.go b/internal/datasource/connector/rss/connector_test.go index ff74c7db..c7771a0f 100644 --- a/internal/datasource/connector/rss/connector_test.go +++ b/internal/datasource/connector/rss/connector_test.go @@ -413,3 +413,28 @@ func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) { t.Fatalf("expected empty ancestors, got %v", got) } } + +func TestResolveItem_FileNamePolicy(t *testing.T) { + for _, tt := range []struct{ name, title, want string }{ + {"empty", "", "untitled.md"}, + {"blank", " \n\t\u2003", "untitled.md"}, + {"line breaks and punctuation", "\u2003 A\nB\rC\tD/E \u2003", "A B C D_E.md"}, + {"existing extension", "Title.md", "Title.md.md"}, + {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"}, + {"space at truncation boundary", strings.Repeat("a", 199) + "\nmore", strings.Repeat("a", 199) + " .md"}, + {"other controls retained", "a\x01b", "a\x01b.md"}, + } { + t.Run(tt.name, func(t *testing.T) { + result := NewConnector().resolveItem( + context.Background(), nil, &gofeed.Feed{}, &gofeed.Item{Title: tt.title}, + "https://example.com/feed", "id", "

content

", + ) + if result.item.FileName != tt.want { + t.Fatalf("FileName = %q, want %q", result.item.FileName, tt.want) + } + if strings.TrimSpace(tt.title) != "" && result.item.Title != tt.title { + t.Fatalf("source title changed: %q", result.item.Title) + } + }) + } +} diff --git a/internal/datasource/connector/rss/types.go b/internal/datasource/connector/rss/types.go index b6f67264..9fff63b5 100644 --- a/internal/datasource/connector/rss/types.go +++ b/internal/datasource/connector/rss/types.go @@ -29,7 +29,6 @@ import ( "maps" "strings" "time" - "unicode/utf8" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/types" @@ -218,32 +217,8 @@ func firstNonEmpty(values ...string) string { return "" } -// sanitizeFileName removes characters invalid in filenames and truncates to a -// safe length at a UTF-8 rune boundary (mirrors the Yuque connector). +// sanitizeFileName keeps feed titles on one line before applying shared limits. func sanitizeFileName(name string) string { - name = strings.TrimSpace(name) - if name == "" { - return "untitled" - } - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - "\n", " ", "\r", " ", "\t", " ", - ) - result := strings.TrimSpace(replacer.Replace(name)) - if result == "" { - return "untitled" - } - const maxBytes = 200 - if len(result) > maxBytes { - result = result[:maxBytes] - for len(result) > 0 { - r, size := utf8.DecodeLastRuneInString(result) - if r != utf8.RuneError || size != 1 { - break - } - result = result[:len(result)-1] - } - } - return result + name = strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(name) + return datasource.SanitizeFileName(strings.TrimSpace(name)) } diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.go index d07fce0c..485eef13 100644 --- a/internal/datasource/connector/yuque/connector.go +++ b/internal/datasource/connector/yuque/connector.go @@ -262,7 +262,7 @@ func (c *Connector) walk( Title: d.Title, Content: []byte(detail.Body), ContentType: "text/markdown", - FileName: sanitizeFileName(d.Title) + ".md", + FileName: datasource.SanitizeFileName(d.Title) + ".md", URL: buildDocURL(cfg.GetBaseURL(), detail.Book.Namespace, d.Slug), UpdatedAt: parseContentUpdatedAt(d.ContentUpdatedAt), SourceResourceID: bookIDStr, diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.go index 44f3ca04..8fc60900 100644 --- a/internal/datasource/connector/yuque/connector_test.go +++ b/internal/datasource/connector/yuque/connector_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/Tencent/WeKnora/internal/types" @@ -484,3 +485,31 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) { t.Errorf("expected Mine + OK Book, got %v", names) } } + +func TestConnector_FetchAll_FileNamePolicy(t *testing.T) { + for _, tt := range []struct{ name, title, want string }{ + {"punctuation and extension", " \tA/B.md ", " \tA_B.md .md"}, + {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"}, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeYuque() + defer f.Close() + f.handleJSON("/api/v2/repos/7/docs", 200, v2DocListResponse{Data: []v2Doc{{ + ID: 101, Type: "Doc", Status: "1", Title: tt.title, ContentUpdatedAt: "2026-04-20T10:00:00Z", + }}}) + f.handleJSON("/api/v2/repos/docs/101", 200, v2DocDetailResponse{Data: v2DocDetail{ + ID: 101, Title: tt.title, Body: "# content", Format: "markdown", Status: "1", + }}) + items, err := NewConnector().FetchAll(context.Background(), makeDSConfig(f, []string{"7"}), []string{"7"}) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 { + t.Fatalf("got %d items, want 1", len(items)) + } + if items[0].Title != tt.title || items[0].FileName != tt.want { + t.Fatalf("title = %q, filename = %q; want %q, %q", items[0].Title, items[0].FileName, tt.title, tt.want) + } + }) + } +} diff --git a/internal/datasource/connector/yuque/types.go b/internal/datasource/connector/yuque/types.go index 63d878d5..b57b7ad2 100644 --- a/internal/datasource/connector/yuque/types.go +++ b/internal/datasource/connector/yuque/types.go @@ -23,7 +23,6 @@ import ( "strconv" "strings" "time" - "unicode/utf8" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/types" @@ -225,37 +224,6 @@ func parseContentUpdatedAt(ts string) time.Time { return t } -// sanitizeFileName removes characters that are invalid in filenames and -// truncates to a safe length at a UTF-8 rune boundary. Raw byte truncation -// would split a multi-byte codepoint (Chinese characters are 3 bytes in UTF-8) -// and produce an invalid UTF-8 string, which downstream filename validation -// (utf8.ValidString) rejects with "文件名包含非法字符". -func sanitizeFileName(name string) string { - if name == "" { - return "untitled" - } - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - ) - result := replacer.Replace(name) - const maxBytes = 200 - if len(result) > maxBytes { - result = result[:maxBytes] - // Peel any trailing bytes that no longer form a complete rune. - // DecodeLastRuneInString returns (RuneError, 1) for a partial UTF-8 - // sequence — at most 3 bytes need peeling since a rune is ≤ 4 bytes. - for len(result) > 0 { - r, size := utf8.DecodeLastRuneInString(result) - if r != utf8.RuneError || size != 1 { - break - } - result = result[:len(result)-1] - } - } - return result -} - // redactToken returns a masked form of the token for logging (never log the full token). func redactToken(t string) string { if len(t) < 12 { diff --git a/internal/datasource/connector/yuque/types_test.go b/internal/datasource/connector/yuque/types_test.go index 1cf6510c..80ede4d4 100644 --- a/internal/datasource/connector/yuque/types_test.go +++ b/internal/datasource/connector/yuque/types_test.go @@ -2,9 +2,7 @@ package yuque import ( "encoding/json" - "strings" "testing" - "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" ) @@ -157,20 +155,3 @@ func TestGetBaseURL_TrimsWhitespace(t *testing.T) { t.Errorf("GetBaseURL = %q, want https://x.yuque.com", got) } } - -func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) { - // Long Chinese title (each 测 is 3 bytes in UTF-8). Raw byte slicing at 200 - // would split a rune and produce invalid UTF-8 that downstream filename - // validation rejects with "文件名包含非法字符". - long := strings.Repeat("测试", 100) // 600 bytes - got := sanitizeFileName(long) - if !utf8.ValidString(got) { - t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got) - } - if len(got) > 200 { - t.Errorf("len = %d, want ≤ 200", len(got)) - } - if len(got) == 0 { - t.Error("result is empty") - } -} diff --git a/internal/datasource/filename.go b/internal/datasource/filename.go new file mode 100644 index 00000000..79e5953b --- /dev/null +++ b/internal/datasource/filename.go @@ -0,0 +1,34 @@ +package datasource + +import ( + "strings" + "unicode/utf8" +) + +var fileNameReplacer = strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", +) + +// SanitizeFileName replaces filesystem-hostile punctuation and limits a source +// title to 200 bytes without splitting a UTF-8 rune. Empty titles use "untitled". +// Whitespace, control characters and extension handling remain the caller's policy; +// this helper does not validate filenames or repair malformed input. +func SanitizeFileName(name string) string { + if name == "" { + return "untitled" + } + result := fileNameReplacer.Replace(name) + const maxBytes = 200 + if len(result) > maxBytes { + result = result[:maxBytes] + for len(result) > 0 { + r, size := utf8.DecodeLastRuneInString(result) + if r != utf8.RuneError || size != 1 { + break + } + result = result[:len(result)-1] + } + } + return result +} diff --git a/internal/datasource/filename_test.go b/internal/datasource/filename_test.go new file mode 100644 index 00000000..2974a7db --- /dev/null +++ b/internal/datasource/filename_test.go @@ -0,0 +1,42 @@ +package datasource + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestSanitizeFileName(t *testing.T) { + cases := []struct{ name, input, want string }{ + {"empty", "", "untitled"}, + {"punctuation", `a/b\c:d*e?f"gi|j`, "a_b_c_d_e_f_g_h_i_j"}, + {"whitespace retained", " \t标题\n\r ", " \t标题\n\r "}, + {"dots retained", "..", ".."}, + {"controls retained", "a\x00b\x1fc", "a\x00b\x1fc"}, + {"ascii at limit", strings.Repeat("a", 200), strings.Repeat("a", 200)}, + {"ascii over limit", strings.Repeat("a", 201), strings.Repeat("a", 200)}, + {"CJK boundary", strings.Repeat("测", 67), strings.Repeat("测", 66)}, + {"two byte boundary", strings.Repeat("a", 199) + "é", strings.Repeat("a", 199)}, + {"emoji boundary", strings.Repeat("a", 198) + "🙂", strings.Repeat("a", 198)}, + {"complete emoji", strings.Repeat("🙂", 51), strings.Repeat("🙂", 50)}, + {"replacement rune", strings.Repeat("a", 197) + "�x", strings.Repeat("a", 197) + "�"}, + {"no extension reservation", strings.Repeat("a", 200) + ".pdf", strings.Repeat("a", 200)}, + {"malformed short input retained", "a\xff", "a\xff"}, + {"malformed interior retained", "\xff" + strings.Repeat("a", 200), "\xff" + strings.Repeat("a", 199)}, + {"malformed cut boundary removed", strings.Repeat("a", 199) + "\xffx", strings.Repeat("a", 199)}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got := SanitizeFileName(tt.input) + if got != tt.want { + t.Fatalf("SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.want) + } + if len(got) > 200 { + t.Fatalf("result exceeds 200 bytes: %d", len(got)) + } + if utf8.ValidString(tt.input) && !utf8.ValidString(got) { + t.Fatalf("split a UTF-8 rune: %q", got) + } + }) + } +}