refactor(datasource): share connector filename sanitization
parent
5780cb0a71
commit
820a14dafa
|
|
@ -98,6 +98,16 @@
|
|||
- 本批只删除无调用代码,复用现有包测试和编译检查,没有添加仅断言函数不存在的测试。
|
||||
- 推送检查暴露新合入的 fork 测试未隔离 Git hook 环境:继承的 `GIT_DIR` 等变量使临时仓库操作指向当前仓库。测试辅助函数现清除继承的 `GIT_*` 变量;真实仓库测试主动注入指向临时路径的仓库变量,验证旧实现失败、修复后通过。该测试修复单独提交,不改变生产逻辑。
|
||||
|
||||
## 第六批已实施:外部来源文件名清理
|
||||
|
||||
基于第五批合入后的 `10973ed32`,IMA、语雀、RSS 复用 `datasource.SanitizeFileName`,统一无效标点替换、空标题回退和 200 字节 UTF-8 边界截断。计入新增共享实现后,生产 Go 源码净减少 49 行。
|
||||
|
||||
- IMA 的媒体与笔记路径、语雀文档直接调用共享函数;RSS 仅保留换行/制表符转空格和首尾空白清理,再调用共享函数。
|
||||
- 保留原有扩展名拼接、源标题、空白、控制字符和无效 UTF-8 输入处理行为。飞书的扩展名保留、钉钉的控制字符/尾部标点清理、Confluence 的按字符截断规则不同,本批未合并。
|
||||
- 将两份旧 helper 测试收敛到共享函数测试,补充 16 个边界用例;在实际连接器读取/组装路径新增 15 个用例,覆盖 IMA 两类内容、语雀和 RSS 的最终文件名。
|
||||
- 用重构前的三个实现对 30,000 组确定性生成输入执行差异校验,结果一致;输入包含中文、emoji、空白、非法字节和截断边界。临时对比代码未留在仓库,避免长期维护旧实现副本。
|
||||
- 4 个受影响包测试通过:104 项顶层测试,包含既有同步、游标和错误处理测试。未修改前端,未运行浏览器检查或真实外部来源服务测试。
|
||||
|
||||
## 后续优先级
|
||||
|
||||
| 优先级 | 证据 | 建议边界 | 验证重点 |
|
||||
|
|
@ -113,7 +123,7 @@
|
|||
### 后端重复的具体判断
|
||||
|
||||
- Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。第五批已删除 Weaviate 的失效函数,保留 Qdrant 实现。
|
||||
- IMA、语雀的 `sanitizeFileName` 函数体相同,可以共享;RSS 的近似实现另外处理首尾空白和换行,合并时必须保留这些差异。
|
||||
- 第六批已合并 IMA、语雀、RSS 的文件名标点替换和 UTF-8 截断逻辑;RSS 的空白预处理保留在连接器内。飞书、钉钉、Confluence 的其他规则继续独立维护。
|
||||
- Milvus/Qdrant/Weaviate 的结果映射体相似,但接收不同后端类型。当前各约 17 行,为此引入通用接口或泛型转换层未必减少维护成本。
|
||||
- 终端和桌面 WebSocket ticket handler 重复检查会话、用户、租户和 token,但签发方式不同。后续可共享前置身份校验,保留 ticket 和连接生命周期各自的契约。
|
||||
- 资源权限已经有 `internal/application/access`;继续沿已有授权边界收敛,避免新建与其并行的权限工具层。
|
||||
|
|
@ -152,4 +162,4 @@
|
|||
- 构建仍报告大 chunk 提示;本轮未以 bundle 体积为目标,也没有测量构建前后的体积差异。
|
||||
- 未修改后端,未运行后端测试;未做浏览器视觉验收。
|
||||
|
||||
下一批优先处理 IMA、语雀等外部来源的重复文件名清理逻辑,核实调用与差异后再决定共享边界。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。
|
||||
下一批优先检查终端与桌面 WebSocket ticket 的重复身份校验,沿现有授权边界确定共享范围。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
|
|
@ -437,7 +436,7 @@ func fetchNote(
|
|||
return types.FetchedItem{}, fetchSkipped
|
||||
}
|
||||
|
||||
fileName := sanitizeFileName(f.Title)
|
||||
fileName := datasource.SanitizeFileName(f.Title)
|
||||
if !strings.HasSuffix(strings.ToLower(fileName), ".md") {
|
||||
fileName += ".md"
|
||||
}
|
||||
|
|
@ -540,7 +539,7 @@ func fetchOneMedia(
|
|||
ct = mimeForExtension(ext)
|
||||
}
|
||||
|
||||
fileName := sanitizeFileName(f.Title)
|
||||
fileName := datasource.SanitizeFileName(f.Title)
|
||||
if !strings.HasSuffix(strings.ToLower(fileName), "."+ext) {
|
||||
fileName = fileName + "." + ext
|
||||
}
|
||||
|
|
@ -585,28 +584,3 @@ func baseMetadata(
|
|||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// sanitizeFileName removes filesystem-hostile characters and truncates to a
|
||||
// safe UTF-8 boundary.
|
||||
func sanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
result := replacer.Replace(name)
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -509,3 +509,35 @@ func TestConnectorIsRegisteredInMetadata(t *testing.T) {
|
|||
t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_FileNamePolicy(t *testing.T) {
|
||||
for _, media := range []struct {
|
||||
name string
|
||||
kind int32
|
||||
}{{"media", mediaTypeMarkdown}, {"note", mediaTypeNote}} {
|
||||
for _, tt := range []struct{ name, title, want string }{
|
||||
{"punctuation", "A/B:C", "A_B_C.md"},
|
||||
{"existing extension and whitespace", " \tA/B.MD", " \tA_B.MD"},
|
||||
{"long title", strings.Repeat("测", 100) + ".MD", strings.Repeat("测", 66) + ".md"},
|
||||
} {
|
||||
t.Run(media.name+"/"+tt.name, func(t *testing.T) {
|
||||
f := newFakeIMA(t)
|
||||
f.setKB("kb1", []fakeFile{{
|
||||
MediaID: "file", Title: tt.title, MediaType: media.kind,
|
||||
Body: "# content", NotebookID: "123", NoteBody: "# content",
|
||||
}})
|
||||
items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("got %d items, want 1", len(items))
|
||||
}
|
||||
if items[0].Title != tt.title || items[0].FileName != tt.want {
|
||||
t.Fatalf("title = %q, filename = %q; want %q, %q",
|
||||
items[0].Title, items[0].FileName, tt.title, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -131,25 +131,6 @@ func TestExtensionForContentType(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFileName(t *testing.T) {
|
||||
if got := sanitizeFileName(""); got != "untitled" {
|
||||
t.Errorf("empty name = %q, want untitled", got)
|
||||
}
|
||||
if got := sanitizeFileName(`a/b\c:d*e?f"g<h>i|j`); strings.ContainsAny(got, `/\:*?"<>|`) {
|
||||
t.Errorf("filesystem-hostile characters survived: %q", got)
|
||||
}
|
||||
|
||||
// A long CJK title must be truncated on a rune boundary, not mid-sequence.
|
||||
long := strings.Repeat("知识", 200)
|
||||
got := sanitizeFileName(long)
|
||||
if len(got) > 200 {
|
||||
t.Errorf("len = %d bytes, want <= 200", len(got))
|
||||
}
|
||||
if !strings.HasPrefix(long, got) {
|
||||
t.Errorf("truncation corrupted the prefix: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSkippableMediaType(t *testing.T) {
|
||||
for _, mt := range []int32{mediaTypeAISession, mediaTypeVideo} {
|
||||
if !isSkippableMediaType(mt) {
|
||||
|
|
|
|||
|
|
@ -413,3 +413,28 @@ func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
|
|||
t.Fatalf("expected empty ancestors, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveItem_FileNamePolicy(t *testing.T) {
|
||||
for _, tt := range []struct{ name, title, want string }{
|
||||
{"empty", "", "untitled.md"},
|
||||
{"blank", " \n\t\u2003", "untitled.md"},
|
||||
{"line breaks and punctuation", "\u2003 A\nB\rC\tD/E \u2003", "A B C D_E.md"},
|
||||
{"existing extension", "Title.md", "Title.md.md"},
|
||||
{"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},
|
||||
{"space at truncation boundary", strings.Repeat("a", 199) + "\nmore", strings.Repeat("a", 199) + " .md"},
|
||||
{"other controls retained", "a\x01b", "a\x01b.md"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := NewConnector().resolveItem(
|
||||
context.Background(), nil, &gofeed.Feed{}, &gofeed.Item{Title: tt.title},
|
||||
"https://example.com/feed", "id", "<p>content</p>",
|
||||
)
|
||||
if result.item.FileName != tt.want {
|
||||
t.Fatalf("FileName = %q, want %q", result.item.FileName, tt.want)
|
||||
}
|
||||
if strings.TrimSpace(tt.title) != "" && result.item.Title != tt.title {
|
||||
t.Fatalf("source title changed: %q", result.item.Title)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import (
|
|||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -218,32 +217,8 @@ func firstNonEmpty(values ...string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters invalid in filenames and truncates to a
|
||||
// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
|
||||
// sanitizeFileName keeps feed titles on one line before applying shared limits.
|
||||
func sanitizeFileName(name string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
"\n", " ", "\r", " ", "\t", " ",
|
||||
)
|
||||
result := strings.TrimSpace(replacer.Replace(name))
|
||||
if result == "" {
|
||||
return "untitled"
|
||||
}
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
name = strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(name)
|
||||
return datasource.SanitizeFileName(strings.TrimSpace(name))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -262,7 +262,7 @@ func (c *Connector) walk(
|
|||
Title: d.Title,
|
||||
Content: []byte(detail.Body),
|
||||
ContentType: "text/markdown",
|
||||
FileName: sanitizeFileName(d.Title) + ".md",
|
||||
FileName: datasource.SanitizeFileName(d.Title) + ".md",
|
||||
URL: buildDocURL(cfg.GetBaseURL(), detail.Book.Namespace, d.Slug),
|
||||
UpdatedAt: parseContentUpdatedAt(d.ContentUpdatedAt),
|
||||
SourceResourceID: bookIDStr,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -484,3 +485,31 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) {
|
|||
t.Errorf("expected Mine + OK Book, got %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_FetchAll_FileNamePolicy(t *testing.T) {
|
||||
for _, tt := range []struct{ name, title, want string }{
|
||||
{"punctuation and extension", " \tA/B.md ", " \tA_B.md .md"},
|
||||
{"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newFakeYuque()
|
||||
defer f.Close()
|
||||
f.handleJSON("/api/v2/repos/7/docs", 200, v2DocListResponse{Data: []v2Doc{{
|
||||
ID: 101, Type: "Doc", Status: "1", Title: tt.title, ContentUpdatedAt: "2026-04-20T10:00:00Z",
|
||||
}}})
|
||||
f.handleJSON("/api/v2/repos/docs/101", 200, v2DocDetailResponse{Data: v2DocDetail{
|
||||
ID: 101, Title: tt.title, Body: "# content", Format: "markdown", Status: "1",
|
||||
}})
|
||||
items, err := NewConnector().FetchAll(context.Background(), makeDSConfig(f, []string{"7"}), []string{"7"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("got %d items, want 1", len(items))
|
||||
}
|
||||
if items[0].Title != tt.title || items[0].FileName != tt.want {
|
||||
t.Fatalf("title = %q, filename = %q; want %q, %q", items[0].Title, items[0].FileName, tt.title, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -225,37 +224,6 @@ func parseContentUpdatedAt(ts string) time.Time {
|
|||
return t
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters that are invalid in filenames and
|
||||
// truncates to a safe length at a UTF-8 rune boundary. Raw byte truncation
|
||||
// would split a multi-byte codepoint (Chinese characters are 3 bytes in UTF-8)
|
||||
// and produce an invalid UTF-8 string, which downstream filename validation
|
||||
// (utf8.ValidString) rejects with "文件名包含非法字符".
|
||||
func sanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
result := replacer.Replace(name)
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
// Peel any trailing bytes that no longer form a complete rune.
|
||||
// DecodeLastRuneInString returns (RuneError, 1) for a partial UTF-8
|
||||
// sequence — at most 3 bytes need peeling since a rune is ≤ 4 bytes.
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// redactToken returns a masked form of the token for logging (never log the full token).
|
||||
func redactToken(t string) string {
|
||||
if len(t) < 12 {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,7 @@ package yuque
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
|
@ -157,20 +155,3 @@ func TestGetBaseURL_TrimsWhitespace(t *testing.T) {
|
|||
t.Errorf("GetBaseURL = %q, want https://x.yuque.com", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) {
|
||||
// Long Chinese title (each 测 is 3 bytes in UTF-8). Raw byte slicing at 200
|
||||
// would split a rune and produce invalid UTF-8 that downstream filename
|
||||
// validation rejects with "文件名包含非法字符".
|
||||
long := strings.Repeat("测试", 100) // 600 bytes
|
||||
got := sanitizeFileName(long)
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got)
|
||||
}
|
||||
if len(got) > 200 {
|
||||
t.Errorf("len = %d, want ≤ 200", len(got))
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Error("result is empty")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
package datasource
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var fileNameReplacer = strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
|
||||
// SanitizeFileName replaces filesystem-hostile punctuation and limits a source
|
||||
// title to 200 bytes without splitting a UTF-8 rune. Empty titles use "untitled".
|
||||
// Whitespace, control characters and extension handling remain the caller's policy;
|
||||
// this helper does not validate filenames or repair malformed input.
|
||||
func SanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
result := fileNameReplacer.Replace(name)
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package datasource
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestSanitizeFileName(t *testing.T) {
|
||||
cases := []struct{ name, input, want string }{
|
||||
{"empty", "", "untitled"},
|
||||
{"punctuation", `a/b\c:d*e?f"g<h>i|j`, "a_b_c_d_e_f_g_h_i_j"},
|
||||
{"whitespace retained", " \t标题\n\r ", " \t标题\n\r "},
|
||||
{"dots retained", "..", ".."},
|
||||
{"controls retained", "a\x00b\x1fc", "a\x00b\x1fc"},
|
||||
{"ascii at limit", strings.Repeat("a", 200), strings.Repeat("a", 200)},
|
||||
{"ascii over limit", strings.Repeat("a", 201), strings.Repeat("a", 200)},
|
||||
{"CJK boundary", strings.Repeat("测", 67), strings.Repeat("测", 66)},
|
||||
{"two byte boundary", strings.Repeat("a", 199) + "é", strings.Repeat("a", 199)},
|
||||
{"emoji boundary", strings.Repeat("a", 198) + "🙂", strings.Repeat("a", 198)},
|
||||
{"complete emoji", strings.Repeat("🙂", 51), strings.Repeat("🙂", 50)},
|
||||
{"replacement rune", strings.Repeat("a", 197) + "<22>x", strings.Repeat("a", 197) + "<22>"},
|
||||
{"no extension reservation", strings.Repeat("a", 200) + ".pdf", strings.Repeat("a", 200)},
|
||||
{"malformed short input retained", "a\xff", "a\xff"},
|
||||
{"malformed interior retained", "\xff" + strings.Repeat("a", 200), "\xff" + strings.Repeat("a", 199)},
|
||||
{"malformed cut boundary removed", strings.Repeat("a", 199) + "\xffx", strings.Repeat("a", 199)},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := SanitizeFileName(tt.input)
|
||||
if got != tt.want {
|
||||
t.Fatalf("SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
if len(got) > 200 {
|
||||
t.Fatalf("result exceeds 200 bytes: %d", len(got))
|
||||
}
|
||||
if utf8.ValidString(tt.input) && !utf8.ValidString(got) {
|
||||
t.Fatalf("split a UTF-8 rune: %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue