diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.go index 4bb0a3d0..cc29835e 100644 --- a/internal/datasource/connector/dingtalk/connector.go +++ b/internal/datasource/connector/dingtalk/connector.go @@ -647,6 +647,12 @@ type syncScope struct { Reference resourceReference StartNodeID string Document *node + // FolderPath is the folder hierarchy between the workspace root and the + // scope's own node, root-first, with the scope's own folder name last. It + // anchors every document the scope yields, so selecting a subfolder and + // selecting the whole workspace file the same document under the same + // folder instead of under two. + FolderPath []string } func (s syncScope) contains(candidate syncScope) bool { @@ -688,6 +694,9 @@ func (c *Connector) sync( api := c.api(cfg) settings := parseDocumentSettings(dataSourceConfig) + if settings.PreserveFolderStructure { + logger.Infof(ctx, "[DingTalk] source folder structure is encoded into document file names") + } workspaces, err := api.listWorkspaces(ctx) if err != nil { return nil, nil, err @@ -696,6 +705,13 @@ func (c *Connector) sync( if err != nil { return nil, nil, err } + // The workspace name is the first folder segment of every encoded path. It + // is read from the listing the sync already made, so the leading level + // costs no extra request. + workspaceFolderNames := make(map[string]string, len(workspaces)) + for _, item := range workspaces { + workspaceFolderNames[item.ID] = workspaceFolderName(item) + } next := &cursorState{ Version: cursorVersion, @@ -757,7 +773,8 @@ func (c *Connector) sync( synced := 0 failed := 0 - for _, document := range documents { + for _, scanned := range documents { + document := scanned.node if document.ID == "" { continue } @@ -770,7 +787,13 @@ func (c *Connector) sync( continue } - item, err := readDocument(ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document) + folders := documentFolders( + settings, workspaceFolderNames[scope.Reference.WorkspaceID], + scope.FolderPath, scanned.folders, + ) + item, err := readDocument( + ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document, folders, + ) if err != nil { if isContextError(err) { return nil, nil, err @@ -894,6 +917,7 @@ func resolveSyncScopes( }, nil } + var ancestorFolders []string parentNodeID := rootNodeID for _, ancestorID := range ref.Ancestors { children, err := listChildren(parentNodeID) @@ -908,6 +932,11 @@ func resolveSyncScopes( if ancestor.WorkspaceID != "" && ancestor.WorkspaceID != ref.WorkspaceID { return syncScope{}, fmt.Errorf("DingTalk ancestor %q belongs to a different workspace", ancestorID) } + // The name is read off a listing this resolution has to make + // anyway, so anchoring the path at the workspace root costs no + // extra request. A selection saved without ancestors yields a + // shallower path rather than an error. + ancestorFolders = append(ancestorFolders, ancestor.title()) parentNodeID = ancestor.ID } children, err := listChildren(parentNodeID) @@ -926,11 +955,13 @@ func resolveSyncScopes( case selectedNode.isFolder(): return syncScope{ ResourceID: canonicalID, Reference: ref, StartNodeID: selectedNode.ID, + FolderPath: appendFolder(ancestorFolders, selectedNode.title()), }, nil case selectedNode.isIngestible(settings): document := selectedNode return syncScope{ ResourceID: canonicalID, Reference: ref, Document: &document, + FolderPath: ancestorFolders, }, nil default: return syncScope{}, fmt.Errorf("DingTalk node %q is not a supported online document or folder", @@ -990,6 +1021,26 @@ func childByID(children []node, nodeID string) (node, bool) { return node{}, false } +// scannedDocument pairs an ingestible node with the folder hierarchy the walk +// found it under, relative to the scan's root node. The hierarchy is carried +// alongside the node rather than recomputed later: the breadth-first walk is +// the only place that knows which parent listed the node. +type scannedDocument struct { + node + // folders holds the folder titles between the scan root and the document, + // root-first. A document sitting directly in the scan root has none. + folders []string +} + +// appendFolder returns parent plus one folder title. The result is always a +// fresh slice: appending in place would let one sibling's name overwrite +// another's inside a shared backing array. +func appendFolder(parent []string, name string) []string { + folders := make([]string, 0, len(parent)+1) + folders = append(folders, parent...) + return append(folders, name) +} + // scanScope lists every ingestible document in one scope, together with the // nodes it saw but cannot ingest. The skipped nodes are returned rather than // discarded so the caller can report them: a full sync must never look clean @@ -999,26 +1050,34 @@ func scanScope( api dingTalkAPI, scope syncScope, settings documentSettings, -) ([]node, []node, error) { +) ([]scannedDocument, []node, error) { if scope.Document != nil { - return []node{*scope.Document}, nil, nil + // A single-document scope has no tree to walk; the scope's own + // FolderPath already carries the folders above it. + return []scannedDocument{{node: *scope.Document}}, nil, nil } return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID, settings) } // scanWorkspace walks a workspace subtree breadth-first. Folders are traversal -// only; every file is either an ingestible document or a skip. +// only; every file is either an ingestible document or a skip. Each document is +// reported with the folder titles the walk descended through to reach it, so +// the caller can mirror the source hierarchy in the knowledge base. func scanWorkspace( ctx context.Context, api dingTalkAPI, workspaceID string, rootNodeID string, settings documentSettings, -) ([]node, []node, error) { +) ([]scannedDocument, []node, error) { queue := []string{rootNodeID} visitedParents := make(map[string]struct{}) seenNodes := make(map[string]struct{}) - var documents []node + // foldersByParent maps a node ID to the folder titles between the scan root + // and that node, root-first. Only nodes the walk descends through get an + // entry; the scan root itself has none. + foldersByParent := map[string][]string{rootNodeID: nil} + var documents []scannedDocument var skipped []node for len(queue) > 0 { @@ -1052,7 +1111,10 @@ func scanWorkspace( } switch { case child.isIngestible(settings): - documents = append(documents, child) + documents = append(documents, scannedDocument{ + node: child, + folders: foldersByParent[parentID], + }) case child.isFolder(): // Containers hold no content of their own, so they are never // reported as skipped. @@ -1060,6 +1122,7 @@ func scanWorkspace( skipped = append(skipped, child) } if child.isFolder() || child.HasChildren { + foldersByParent[child.ID] = appendFolder(foldersByParent[parentID], child.title()) queue = append(queue, child.ID) } } @@ -1141,13 +1204,16 @@ func warnUnknownBlockTypes(ctx context.Context, document node, rendered renderRe // readDocument turns one supported wiki node into a FetchedItem. Native adoc // documents are rendered from their blocks, and uploaded files are transferred -// byte-for-byte so WeKnora's own parsers can read them. +// byte-for-byte so WeKnora's own parsers can read them. folders is the source +// folder hierarchy to encode in the item's FileName, or nil to keep the +// historical flat name. func readDocument( ctx context.Context, api dingTalkAPI, sourceResourceID string, workspaceID string, document node, + folders []string, ) (types.FetchedItem, error) { switch { case document.isOnlineDocument(): @@ -1160,16 +1226,73 @@ func readDocument( // both the request context and the node identity are known. Uploaded // files take the default branch and have no rendering step to lose. warnUnknownBlockTypes(ctx, document, rendered) - return fetchedDocument(sourceResourceID, workspaceID, document, rendered), nil + return fetchedDocument(sourceResourceID, workspaceID, document, rendered, folders), nil default: data, err := api.downloadDocument(ctx, document.ID) if err != nil { return types.FetchedItem{}, err } - return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data), nil + return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data, folders), nil } } +// documentFolders assembles the folder segments a document's FileName encodes: +// the workspace, the folders between the workspace root and the scope's own +// node, then the folders the scan descended through. +// +// It returns nil when the switch is off, which is what keeps the emitted +// FileName byte-for-byte identical to previous releases for every data source +// that never turned it on. +func documentFolders( + settings documentSettings, + workspaceName string, + scopeFolders []string, + scanFolders []string, +) []string { + if !settings.PreserveFolderStructure { + return nil + } + folders := make([]string, 0, len(scopeFolders)+len(scanFolders)+1) + if workspaceName != "" { + folders = append(folders, workspaceName) + } + folders = append(folders, scopeFolders...) + return append(folders, scanFolders...) +} + +// workspaceFolderName names a workspace the way the resource picker does: the +// workspace name, falling back to its ID, which is what an operator sees when a +// tenant exposes no name. +func workspaceFolderName(item workspace) string { + if name := strings.TrimSpace(item.Name); name != "" { + return name + } + return item.ID +} + +// buildFolderFileName prepends the folder hierarchy to a document's file name. +// +// folder_path downstream is derived from FileName alone +// (SplitKnowledgeRelativePath → NormalizeKnowledgeFolderPath), so the hierarchy +// has to be encoded here as a relative path. +// +// Every level is sanitised on its own, as the Yuque connector's own folder +// encoding does, so a "/" inside a folder name cannot silently create an extra +// nesting level. Folder names the downstream normaliser rejects outright — ".", +// "..", an empty name — drop their level instead of failing the sync. An empty +// folder list, which is what the default settings produce, returns the file +// name unchanged. +func buildFolderFileName(folders []string, fileName string) string { + if len(folders) == 0 { + return fileName + } + segments := make([]string, 0, len(folders)) + for _, folder := range folders { + segments = append(segments, datasource.SanitizeFileName(folder)) + } + return strings.Join(segments, "/") + "/" + fileName +} + // documentMetadata describes a wiki node the same way for rendered and // downloaded documents, so both stay searchable by their DingTalk identity. func documentMetadata(workspaceID string, document node) map[string]string { @@ -1195,6 +1318,7 @@ func fetchedDocument( workspaceID string, document node, rendered renderResult, + folders []string, ) types.FetchedItem { metadata := documentMetadata(workspaceID, document) if len(rendered.UnknownTypes) > 0 { @@ -1205,7 +1329,7 @@ func fetchedDocument( Title: document.title(), Content: []byte(rendered.Markdown), ContentType: "text/markdown", - FileName: renderedDocumentFileName(document.title()), + FileName: buildFolderFileName(folders, renderedDocumentFileName(document.title())), URL: documentURL(document), UpdatedAt: document.modifiedAt(), Metadata: metadata, @@ -1234,6 +1358,7 @@ func fetchedBinaryDocument( workspaceID string, document node, data []byte, + folders []string, ) types.FetchedItem { contentType, ok := document.binaryContentType() if !ok { @@ -1244,7 +1369,7 @@ func fetchedBinaryDocument( Title: document.title(), Content: data, ContentType: contentType, - FileName: binaryDocumentFileName(document), + FileName: buildFolderFileName(folders, binaryDocumentFileName(document)), URL: documentURL(document), UpdatedAt: document.modifiedAt(), Metadata: documentMetadata(workspaceID, document), diff --git a/internal/datasource/connector/dingtalk/folder_path_settings_test.go b/internal/datasource/connector/dingtalk/folder_path_settings_test.go new file mode 100644 index 00000000..90e03701 --- /dev/null +++ b/internal/datasource/connector/dingtalk/folder_path_settings_test.go @@ -0,0 +1,198 @@ +package dingtalk + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// --- settings --- + +// The folder switch is read through the same helper as the uploaded-files +// switch, and an absent, malformed or unexpected value has to leave the data +// source on the historical flat layout. +func TestParseDocumentSettingsReadsTheFolderSwitch(t *testing.T) { + for _, testCase := range []struct { + label string + settings map[string]interface{} + want bool + }{ + {"absent key", map[string]interface{}{}, false}, + {"null settings entry", map[string]interface{}{"preserve_folder_structure": nil}, false}, + {"bool true", map[string]interface{}{"preserve_folder_structure": true}, true}, + {"bool false", map[string]interface{}{"preserve_folder_structure": false}, false}, + {"numeric string", map[string]interface{}{"preserve_folder_structure": "1"}, true}, + {"padded on", map[string]interface{}{"preserve_folder_structure": " on "}, true}, + {"padded OFF", map[string]interface{}{"preserve_folder_structure": " OFF "}, false}, + {"unknown spelling", map[string]interface{}{"preserve_folder_structure": "maybe"}, false}, + {"unexpected type", map[string]interface{}{"preserve_folder_structure": 1}, false}, + { + "independent of the uploaded-files switch", + map[string]interface{}{"include_uploaded_files": true}, + false, + }, + } { + t.Run(testCase.label, func(t *testing.T) { + dataSourceConfig := testConfig("ws") + dataSourceConfig.Settings = testCase.settings + got := parseDocumentSettings(dataSourceConfig) + if got.PreserveFolderStructure != testCase.want { + t.Fatalf("PreserveFolderStructure = %t, want %t", got.PreserveFolderStructure, testCase.want) + } + if got.IncludeUploadedFiles != settingBool(testCase.settings["include_uploaded_files"], false) { + t.Fatalf("IncludeUploadedFiles = %t, the two switches are not independent", + got.IncludeUploadedFiles) + } + }) + } + if got := parseDocumentSettings(nil); got.PreserveFolderStructure { + t.Fatal("parseDocumentSettings(nil).PreserveFolderStructure = true, want false") + } +} + +// documentFolders is the single place the switch is honoured, so the nil it +// returns when the switch is off is what keeps an existing data source +// byte-for-byte unchanged. +func TestDocumentFoldersIsNilUnlessTheSwitchIsOn(t *testing.T) { + scope := []string{"天诛渗透攻防"} + scan := []string{"2024 年度"} + off := documentFolders(documentSettings{}, "工作区", scope, scan) + if off != nil { + t.Fatalf("documentFolders() with the switch off = %#v, want nil", off) + } + on := documentFolders( + documentSettings{PreserveFolderStructure: true}, "工作区", scope, scan, + ) + want := []string{"工作区", "天诛渗透攻防", "2024 年度"} + if strings.Join(on, "|") != strings.Join(want, "|") { + t.Fatalf("documentFolders() = %#v, want %#v", on, want) + } + // A workspace with no name of its own contributes no segment rather than an + // empty level. + unnamed := documentFolders(documentSettings{PreserveFolderStructure: true}, "", scope, nil) + if strings.Join(unnamed, "|") != "天诛渗透攻防" { + t.Fatalf("documentFolders() without a workspace name = %#v", unnamed) + } +} + +// --- file name assembly --- + +// The folder path a synced document ends up in is derived downstream from +// FileName alone, so the encoding is only correct if that split recovers the +// hierarchy. +func TestBuildFolderFileNameSurvivesTheDownstreamFolderSplit(t *testing.T) { + for _, testCase := range []struct { + label string + folders []string + fileName string + wantFolder string + wantName string + }{ + { + "no folders keeps the historical shape", + nil, "FAQ.md", "", "FAQ.md", + }, + { + "workspace only", + []string{"工作区"}, + "FAQ.md", "工作区", "FAQ.md", + }, + { + "workspace and nested folders", + []string{"天穹安全能力中心", "天诛渗透攻防", "2024 年度"}, + "2024 年度复盘 FAQ.md", + "天穹安全能力中心/天诛渗透攻防/2024 年度", + "2024 年度复盘 FAQ.md", + }, + { + "a separator inside a name cannot add a level", + []string{"安全/中心", "A/B"}, + "A_B 手册.md", + "安全_中心/A_B", + "A_B 手册.md", + }, + { + "a level the normaliser rejects is dropped, not fatal", + []string{"工作区", ".", "子目录"}, + "FAQ.md", + "工作区/子目录", + "FAQ.md", + }, + } { + t.Run(testCase.label, func(t *testing.T) { + raw := buildFolderFileName(testCase.folders, testCase.fileName) + folder, name := types.SplitKnowledgeRelativePath(raw) + if folder != testCase.wantFolder || name != testCase.wantName { + t.Fatalf("split(%q) = (%q, %q), want (%q, %q)", + raw, folder, name, testCase.wantFolder, testCase.wantName) + } + }) + } +} + +func TestBuildFolderFileNameWithoutFoldersIsTheIdentity(t *testing.T) { + for _, name := range []string{"FAQ.md", "手册.docx", "untitled"} { + if got := buildFolderFileName(nil, name); got != name { + t.Fatalf("buildFolderFileName(nil, %q) = %q, want it unchanged", name, got) + } + } +} + +// Sibling folders are recorded while the walk visits them, so a name must not +// be written into a slice another sibling is still holding. +func TestAppendFolderNeverSharesABackingArray(t *testing.T) { + parent := make([]string, 1, 8) + parent[0] = "工作区" + first := appendFolder(parent, "天诛渗透攻防") + second := appendFolder(parent, "天禹漏洞情报") + if first[len(first)-1] != "天诛渗透攻防" { + t.Fatalf("first sibling = %#v, its name was overwritten", first) + } + if second[len(second)-1] != "天禹漏洞情报" { + t.Fatalf("second sibling = %#v, its name was overwritten", second) + } + if len(parent) != 1 || parent[0] != "工作区" { + t.Fatalf("appendFolder() mutated its input: %#v", parent) + } +} + +// --- cursor safety --- + +// The folder dimension is an ingest-path concern only. If it reached the sync +// cursor, turning the switch on would change what the next incremental sync +// considers unchanged — and, worse, which documents it considers deleted. +func TestFolderPathsNeverReachTheSyncCursor(t *testing.T) { + _, off, err := testConnector(tenantFolderPathFixture()).FetchIncremental( + context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil, + ) + if err != nil { + t.Fatalf("FetchIncremental() with the switch off error = %v", err) + } + _, on, err := testConnector(tenantFolderPathFixture()).FetchIncremental( + context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), nil, + ) + if err != nil { + t.Fatalf("FetchIncremental() with the switch on error = %v", err) + } + encode := func(t *testing.T, cursor *types.SyncCursor) string { + t.Helper() + if cursor == nil { + t.Fatal("FetchIncremental() returned no cursor") + } + raw, err := json.Marshal(cursor.ConnectorCursor["resources"]) + if err != nil { + t.Fatal(err) + } + return string(raw) + } + offCursor, onCursor := encode(t, off), encode(t, on) + if offCursor != onCursor { + t.Fatalf("the cursor depends on the folder switch:\noff: %s\non: %s", offCursor, onCursor) + } + if strings.Contains(onCursor, tenantWorkspaceName) { + t.Fatalf("the cursor carries folder names: %s", onCursor) + } +} diff --git a/internal/datasource/connector/dingtalk/folder_path_test.go b/internal/datasource/connector/dingtalk/folder_path_test.go new file mode 100644 index 00000000..94374d2c --- /dev/null +++ b/internal/datasource/connector/dingtalk/folder_path_test.go @@ -0,0 +1,435 @@ +package dingtalk + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// The fixture below reproduces the shape of the tenant that reported the +// problem: one workspace whose root holds five folders, a document that sits +// directly at the root, both formats inside the folders (native adoc FAQ pages +// and uploaded docx/pptx manuals), an empty folder, and one extra nesting level +// so the depth is not uniform. It goes through the public fetch entry points +// only, so every assertion describes what an operator sees in the knowledge +// base rather than how the connector is wired internally. +// +// Node names carry no file extension: DingTalk reports the extension in its own +// field, exactly as the rest of this package's fixtures do. +const ( + tenantWorkspaceID = "ws-tianqiong" + tenantWorkspaceName = "天穹安全能力中心-天系列使用FAQ" + tenantRootNodeID = "root-tianqiong" + // tenantModifiedAt is the millisecond revision DingTalk reports for the + // fixture's documents. It is what makes an incremental sync skip an + // untouched document, so the relocation case below is about the file name + // and not about a missing revision. + tenantModifiedAt = int64(1760000000000) +) + +// tenantUploadedFilesConfig is the state of the reported tenant before this +// change: uploaded manuals are ingested, the folder-path switch does not exist +// yet, so the settings bag has no key for it. +func tenantUploadedFilesConfig(resources ...string) *types.DataSourceConfig { + cfg := testConfig(resources...) + cfg.Settings = map[string]interface{}{"include_uploaded_files": true} + return cfg +} + +// tenantFolderPathTestConfig turns the folder-path switch on as well. +func tenantFolderPathTestConfig(resources ...string) *types.DataSourceConfig { + cfg := tenantUploadedFilesConfig(resources...) + cfg.Settings["preserve_folder_structure"] = true + return cfg +} + +// tenantFolderPathFixture builds the reported tenant tree. +func tenantFolderPathFixture() *fakeAPI { + adoc := func(id, name string) node { + return node{ + ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "ALIDOC", + Extension: "adoc", Name: name, ModifiedTimestamp: tenantModifiedAt, + } + } + upload := func(id, name, extension string) node { + return node{ + ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "DOCUMENT", + Extension: extension, Name: name, ModifiedTimestamp: tenantModifiedAt, + } + } + folder := func(id, name string) node { + return node{ + ID: id, WorkspaceID: tenantWorkspaceID, Type: "FOLDER", Name: name, HasChildren: true, + } + } + return &fakeAPI{ + workspaces: []workspace{{ + ID: tenantWorkspaceID, RootNodeID: tenantRootNodeID, Name: tenantWorkspaceName, + }}, + nodes: map[string][]node{ + tenantRootNodeID: { + folder("f-easm", "EASM-外部攻击面管理模块"), + adoc("d-cloud-faq", "天穹云端安全能力中枢常见问题 FAQ"), + folder("f-tianxi", "天析应急分析"), + folder("f-tianyu", "天禹漏洞情报"), + folder("f-tianfu", "天伏攻防知识库"), + folder("f-tianzhu", "天诛渗透攻防"), + }, + "f-easm": { + adoc("d-easm-faq", "EASM 模块使用 FAQ"), + upload("d-easm-manual", "EASM 模块操作手册.docx", "docx"), + }, + "f-tianxi": { + adoc("d-tianxi-faq", "天析应急分析 FAQ"), + upload("d-tianxi-deck", "天析应急分析介绍.pptx", "pptx"), + }, + "f-tianyu": { + upload("d-tianyu-manual", "天禹漏洞情报使用手册.docx", "docx"), + upload("d-tianyu-deck", "天禹漏洞情报介绍.pptx", "pptx"), + }, + // 天伏攻防知识库 is listed but holds nothing: an empty folder must + // stay invisible in the knowledge base instead of producing a level + // with no documents in it. + "f-tianfu": {}, + "f-tianzhu": { + upload("d-tianzhu-manual", "天诛渗透攻防手册.docx", "docx"), + adoc("d-tianzhu-faq-1", "天诛渗透攻防 FAQ 一"), + adoc("d-tianzhu-faq-2", "天诛渗透攻防 FAQ 二"), + folder("f-tianzhu-2024", "2024 年度"), + }, + "f-tianzhu-2024": { + adoc("d-tianzhu-2024-faq", "2024 年度复盘 FAQ"), + }, + }, + blocks: map[string][]json.RawMessage{ + "d-cloud-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"cloud"}}`)}, + "d-easm-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"easm"}}`)}, + "d-tianxi-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"tianxi"}}`)}, + "d-tianzhu-faq-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"one"}}`)}, + "d-tianzhu-faq-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"two"}}`)}, + "d-tianzhu-2024-faq": { + rawJSON(`{"blockType":"paragraph","paragraph":{"text":"review"}}`), + }, + }, + downloads: map[string][]byte{ + "d-easm-manual": []byte("easm"), + "d-tianxi-deck": []byte("tianxi"), + "d-tianyu-manual": []byte("tianyu-manual"), + "d-tianyu-deck": []byte("tianyu-deck"), + "d-tianzhu-manual": []byte("tianzhu"), + }, + nodeErrors: make(map[string]error), + blockErrors: make(map[string]error), + } +} + +// fileNamesByDocument indexes the fetched items by node ID so an assertion can +// name the document it is talking about instead of an index. +func fileNamesByDocument(t *testing.T, items []types.FetchedItem) map[string]string { + t.Helper() + names := make(map[string]string, len(items)) + for _, item := range items { + if _, seen := names[item.ExternalID]; seen { + t.Fatalf("document %q was fetched more than once", item.ExternalID) + } + names[item.ExternalID] = item.FileName + } + return names +} + +func assertFileNames(t *testing.T, got map[string]string, want map[string]string) { + t.Helper() + for id, wantName := range want { + if got[id] != wantName { + t.Errorf("document %s FileName = %q, want %q", id, got[id], wantName) + } + } + if len(got) != len(want) { + t.Fatalf("fetched %d documents, want %d: %#v", len(got), len(want), got) + } +} + +// Regression guard: a data source that never turned the switch on keeps the +// historical flat file names, byte for byte. Every document of the reported +// tenant is listed, in the same format it was ingested with before. +func TestDingTalkFolderPathsDefaultOffKeepsHistoricalFileNames(t *testing.T) { + captureLogs(t) + connector := testConnector(tenantFolderPathFixture()) + items, err := connector.FetchAll( + context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), + []string{tenantWorkspaceID}, + ) + if err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + assertFileNames(t, fileNamesByDocument(t, items), map[string]string{ + "d-easm-faq": "EASM 模块使用 FAQ.md", + "d-easm-manual": "EASM 模块操作手册.docx", + "d-cloud-faq": "天穹云端安全能力中枢常见问题 FAQ.md", + "d-tianxi-faq": "天析应急分析 FAQ.md", + "d-tianxi-deck": "天析应急分析介绍.pptx", + "d-tianyu-manual": "天禹漏洞情报使用手册.docx", + "d-tianyu-deck": "天禹漏洞情报介绍.pptx", + "d-tianzhu-manual": "天诛渗透攻防手册.docx", + "d-tianzhu-faq-1": "天诛渗透攻防 FAQ 一.md", + "d-tianzhu-faq-2": "天诛渗透攻防 FAQ 二.md", + "d-tianzhu-2024-faq": "2024 年度复盘 FAQ.md", + }) +} + +// The reported problem, end to end: with the switch on, every document encodes +// its source folder hierarchy relative to the workspace root, so the five +// top-level folders stay distinguishable in the knowledge base. The workspace +// name is the first segment, and a document sitting at the root carries no +// folder segment beyond it. +func TestDingTalkFolderPathsPreserveTheTenantShapeWhenEnabled(t *testing.T) { + captureLogs(t) + connector := testConnector(tenantFolderPathFixture()) + items, err := connector.FetchAll( + context.Background(), + tenantFolderPathTestConfig(tenantWorkspaceID), + []string{tenantWorkspaceID}, + ) + if err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + root := tenantWorkspaceName + "/" + assertFileNames(t, fileNamesByDocument(t, items), map[string]string{ + "d-easm-faq": root + "EASM-外部攻击面管理模块/EASM 模块使用 FAQ.md", + "d-easm-manual": root + "EASM-外部攻击面管理模块/EASM 模块操作手册.docx", + "d-cloud-faq": root + "天穹云端安全能力中枢常见问题 FAQ.md", + "d-tianxi-faq": root + "天析应急分析/天析应急分析 FAQ.md", + "d-tianxi-deck": root + "天析应急分析/天析应急分析介绍.pptx", + "d-tianyu-manual": root + "天禹漏洞情报/天禹漏洞情报使用手册.docx", + "d-tianyu-deck": root + "天禹漏洞情报/天禹漏洞情报介绍.pptx", + "d-tianzhu-manual": root + "天诛渗透攻防/天诛渗透攻防手册.docx", + "d-tianzhu-faq-1": root + "天诛渗透攻防/天诛渗透攻防 FAQ 一.md", + "d-tianzhu-faq-2": root + "天诛渗透攻防/天诛渗透攻防 FAQ 二.md", + "d-tianzhu-2024-faq": root + "天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md", + }) +} + +// The switch has to read the same string spellings the connector's other switch +// reads, and anything unrecognised — including an absent key — has to leave the +// historical behaviour alone. +func TestDingTalkFolderPathsReadsTheSameSettingsSpellings(t *testing.T) { + captureLogs(t) + for _, testCase := range []struct { + label string + settings map[string]interface{} + want string + }{ + { + "key absent", + map[string]interface{}{"include_uploaded_files": true}, + "天诛渗透攻防 FAQ 一.md", + }, + { + "bool true", + map[string]interface{}{"preserve_folder_structure": true}, + tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md", + }, + { + "string yes", + map[string]interface{}{"preserve_folder_structure": "yes"}, + tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md", + }, + { + "string ON with padding", + map[string]interface{}{"preserve_folder_structure": " ON "}, + tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md", + }, + { + "bool false", + map[string]interface{}{"preserve_folder_structure": false}, + "天诛渗透攻防 FAQ 一.md", + }, + { + "unknown spelling", + map[string]interface{}{"preserve_folder_structure": "sometimes"}, + "天诛渗透攻防 FAQ 一.md", + }, + { + "unexpected type", + map[string]interface{}{"preserve_folder_structure": 42}, + "天诛渗透攻防 FAQ 一.md", + }, + } { + t.Run(testCase.label, func(t *testing.T) { + cfg := tenantUploadedFilesConfig(tenantWorkspaceID) + cfg.Settings = testCase.settings + connector := testConnector(tenantFolderPathFixture()) + items, err := connector.FetchAll( + context.Background(), cfg, []string{tenantWorkspaceID}, + ) + if err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + names := fileNamesByDocument(t, items) + if names["d-tianzhu-faq-1"] != testCase.want { + t.Fatalf("FileName = %q, want %q", names["d-tianzhu-faq-1"], testCase.want) + } + }) + } +} + +// Selecting a folder is the common case, and the path a document lands on must +// not depend on how deep the operator drilled into the picker: the same +// document selected through the whole workspace, through its own folder and as +// a single document has to end up in one folder, not three. +func TestDingTalkFolderPathsDoNotDependOnTheSelectedScope(t *testing.T) { + captureLogs(t) + connector := testConnector(tenantFolderPathFixture()) + wholeWorkspace, err := connector.FetchAll( + context.Background(), + tenantFolderPathTestConfig(tenantWorkspaceID), + []string{tenantWorkspaceID}, + ) + if err != nil { + t.Fatalf("workspace scope error = %v", err) + } + folderID, err := encodeResourceReference(resourceReference{ + WorkspaceID: tenantWorkspaceID, NodeID: "f-tianzhu-2024", Ancestors: []string{"f-tianzhu"}, + }) + if err != nil { + t.Fatal(err) + } + nestedFolder, err := connector.FetchAll( + context.Background(), + tenantFolderPathTestConfig(folderID), + []string{folderID}, + ) + if err != nil { + t.Fatalf("folder scope error = %v", err) + } + documentID, err := encodeResourceReference(resourceReference{ + WorkspaceID: tenantWorkspaceID, NodeID: "d-tianzhu-2024-faq", + Ancestors: []string{"f-tianzhu", "f-tianzhu-2024"}, + }) + if err != nil { + t.Fatal(err) + } + singleDocument, err := connector.FetchAll( + context.Background(), + tenantFolderPathTestConfig(documentID), + []string{documentID}, + ) + if err != nil { + t.Fatalf("document scope error = %v", err) + } + + want := tenantWorkspaceName + "/天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md" + for label, items := range map[string][]types.FetchedItem{ + "workspace": wholeWorkspace, + "folder": nestedFolder, + "document": singleDocument, + } { + if got := fileNamesByDocument(t, items)["d-tianzhu-2024-faq"]; got != want { + t.Errorf("%s scope FileName = %q, want %q", label, got, want) + } + } +} + +// Names are hostile but not tidy: a folder name may contain the path separator, +// punctuation DingTalk allows and file systems do not, and two sibling folders +// may share a name. Each level is sanitised on its own so a "/" inside one name +// cannot invent an extra level, and same-named siblings merge into one folder — +// the same trade-off the Yuque connector documents for same-named groups. +func TestDingTalkFolderPathsSanitizeEachLevelAndMergeSameNamedFolders(t *testing.T) { + captureLogs(t) + const workspaceName = "安全/中心" + adoc := func(id, name string) node { + return node{ + ID: id, WorkspaceID: "ws-odd", Type: "FILE", Category: "ALIDOC", + Extension: "adoc", Name: name, + } + } + folder := func(id, name string) node { + return node{ + ID: id, WorkspaceID: "ws-odd", Type: "FOLDER", Name: name, HasChildren: true, + } + } + api := &fakeAPI{ + workspaces: []workspace{{ID: "ws-odd", RootNodeID: "root-odd", Name: workspaceName}}, + nodes: map[string][]node{ + "root-odd": { + folder("f-slash", "A/B"), + folder("f-punct", `<>:"|?*`), + // Two folders with the same title: DingTalk node IDs are + // unique, titles are not. + folder("f-dup-1", "重复"), + folder("f-dup-2", "重复"), + // A name of only whitespace falls back to the node ID, which + // is what the resource picker shows for it too. + folder("f-unnamed", " "), + }, + "f-slash": {adoc("d-slash", "A/B 手册")}, + "f-punct": {adoc("d-punct", "符号")}, + "f-dup-1": {adoc("d-dup-1", "第一份")}, + "f-dup-2": {adoc("d-dup-2", "第二份")}, + "f-unnamed": {adoc("d-unnamed", "无名目录")}, + }, + blocks: map[string][]json.RawMessage{ + "d-slash": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"a"}}`)}, + "d-punct": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"b"}}`)}, + "d-dup-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"c"}}`)}, + "d-dup-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"d"}}`)}, + "d-unnamed": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"e"}}`)}, + }, + nodeErrors: make(map[string]error), + blockErrors: make(map[string]error), + } + connector := testConnector(api) + items, err := connector.FetchAll( + context.Background(), tenantFolderPathTestConfig("ws-odd"), []string{"ws-odd"}, + ) + if err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + // "安全/中心" cannot invent a level either, and neither can "A/B" or the + // document name: the file name keeps the sanitised form it already had. + assertFileNames(t, fileNamesByDocument(t, items), map[string]string{ + "d-slash": "安全_中心/A_B/A_B 手册.md", + "d-punct": "安全_中心/_______/符号.md", + "d-dup-1": "安全_中心/重复/第一份.md", + "d-dup-2": "安全_中心/重复/第二份.md", + "d-unnamed": "安全_中心/f-unnamed/无名目录.md", + }) +} + +// Turning the switch on in an existing data source is an ingest-path change, +// not a migration: an incremental sync compares content revisions, so the +// documents already in the knowledge base are not re-read and therefore keep +// the flat file names they were ingested with — and, above all, the folder +// dimension must never make the connector report them as deleted. +func TestDingTalkEnablingFolderPathsDoesNotRelocateOrDeleteExistingDocuments(t *testing.T) { + captureLogs(t) + connector := testConnector(tenantFolderPathFixture()) + items, cursor, err := connector.FetchIncremental( + context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil, + ) + if err != nil { + t.Fatalf("first FetchIncremental() error = %v", err) + } + if names := fileNamesByDocument(t, items); names["d-tianzhu-faq-1"] != "天诛渗透攻防 FAQ 一.md" { + t.Fatalf("first sync FileName = %q, want the flat name", names["d-tianzhu-faq-1"]) + } + + items, _, err = connector.FetchIncremental( + context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), cursor, + ) + if err != nil { + t.Fatalf("second FetchIncremental() error = %v", err) + } + for _, item := range items { + if item.IsDeleted { + t.Fatalf("enabling folder paths reported document %q as deleted", item.ExternalID) + } + } + if len(items) != 0 { + t.Fatalf("second sync returned %d items, want none: unchanged documents are not "+ + "re-read, so they keep the file name they were ingested with", len(items)) + } +} diff --git a/internal/datasource/connector/dingtalk/settings.go b/internal/datasource/connector/dingtalk/settings.go index 9f224e67..3a530659 100644 --- a/internal/datasource/connector/dingtalk/settings.go +++ b/internal/datasource/connector/dingtalk/settings.go @@ -37,6 +37,14 @@ type documentSettings struct { // IncludeUploadedFiles admits uploaded Office/PDF files (FILE/DOCUMENT // nodes) that are downloaded from 钉盘 and parsed by WeKnora itself. IncludeUploadedFiles bool + // PreserveFolderStructure encodes each document's DingTalk folder hierarchy + // into its FileName, so the document lands in the knowledge base folder + // that mirrors where it lives in the wiki instead of at the root. + // + // It is an ingest-path switch, not a migration. An already-ingested + // document is only re-read when its content changes, so turning the switch + // on does not relocate it; a full sync does. + PreserveFolderStructure bool } // parseDocumentSettings reads the settings bag of a data source. A missing or @@ -48,6 +56,7 @@ func parseDocumentSettings(dataSourceConfig *types.DataSourceConfig) documentSet raw = dataSourceConfig.Settings } return documentSettings{ - IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false), + IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false), + PreserveFolderStructure: settingBool(raw["preserve_folder_structure"], false), } } diff --git a/website-docs/03-features/10-datasource.md b/website-docs/03-features/10-datasource.md index 66276dd3..8d90188b 100644 --- a/website-docs/03-features/10-datasource.md +++ b/website-docs/03-features/10-datasource.md @@ -233,6 +233,12 @@ - **上传文件(默认关闭)**:`Settings.include_uploaded_files`(默认 `false`)开启后,所选范围内钉盘中的普通上传附件(`FILE/DOCUMENT` 的 `docx` / `pptx` / `xlsx` / `pdf`)会被下载并交给 WeKnora 自带解析器入库,文件名保留原扩展名、按扩展名选择解析器;单个文件上限 64 MiB,超出上限即失败且不重试(重签名只会拿到同样的字节)。下载失败(401/403/404 等)会重新申请签名后重试,最多 3 次。 - **开关的默认值与成本**:`include_uploaded_files` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后的下一次同步(含增量同步)行为不变**,不会突然开始下载和向量化附件。打开后,这些文件的正文会进入知识库,占用存储并产生 embedding 成本;不需要时请保持关闭。关闭时连接校验、资源列表和同步都会跳过上传附件,同步日志给出的原因是 `include_uploaded_files` 未启用,而不是类型不支持。 - **开关的作用点**:测试连接(校验)、资源选择列表、全量同步与增量同步共用同一个判据,一处关闭即处处一致。 +- **目录结构(默认关闭)**:钉钉知识库本身就是目录树,但连接器历来把文档平铺到知识库根目录,`folder_path` 为空,不同来源目录的文档在知识库里无法区分。`Settings.preserve_folder_structure`(默认 `false`)开启后,`FileName` 编码为 `<工作区名>/<子目录>/…/<标题>.ext`,由下游 `SplitKnowledgeRelativePath` → `NormalizeKnowledgeFolderPath` 拆出 `folder_path`——与语雀 `folder_mode=toc`、飞书云盘目录映射是同一套机制。**工作区名是第一段**:一个知识库可以同时链接多个钉钉数据源,没有这一段,不同工作区里同名的顶层目录会静默合并成一个(语雀用书名作第一段是同样的理由);工作区没有名字时退回用它的 ID。**路径始终以工作区为根**:选整个工作区、选某个子目录、选单篇文档,同一篇文档都落到同一个目录,不会出现两份。每一层目录名单独清洗(名字里的 `/`、`\`、`:` 等替换为 `_`),因此目录名里的斜杠不会凭空多出一层;同名子目录合并为同一个知识库目录(钉钉节点 ID 唯一、标题不唯一),`.`、`..` 这类非法层级由下游归一化丢弃而不是让同步失败。开启后目录由连接器托管:在知识库里手动调整过的目录,会在该文档下次同步时被覆盖回钉钉的结构。 +- **目录开关的默认值与作用范围**:`preserve_folder_structure` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后 `FileName` 逐字节不变、`folder_path` 仍为空**。它只决定**入库路径**,不决定哪些文档入库:测试连接与资源选择列表不受它影响,关掉它也不会让任何文档消失。目录维度不写入同步游标——开关开与关时游标里的"资源 → 文档版本"表完全一致,所以目录永远不会让文档被判定为"源端已删除"。 +- **目录相关注意 / 限制**: + - **开启开关不会自动归位已导入的文档**:增量同步按 `modifiedTimestamp` 判断变化,只重新读取内容有变化的文档,因此**已经在知识库里的文档会保持原来的扁平文件名**——它们并没有被重新读取。只有内容变更过的文档、或手动触发一次全量同步之后,才会整体移到新目录。这与语雀修改 `folder_mode` 后的行为一致。 + - **`include_uploaded_files` 关闭时**,上传附件(`docx` / `pptx` / `xlsx` / `pdf`)不入库,其所在目录自然也不会出现;在线 `adoc` 文档不受该开关影响。 + - **目录深度与长度由下游统一裁剪**:`folder_path` 最多 16 层、单个目录名最多 128 字节、整条路径最多 1024 字节,超出部分被截断(与文件夹上传同一套规则)。 - **同步**:按文档 `modifiedTimestamp`(毫秒)增量读取,缺失时回退 `modifiedTime`;合并重叠选择。全量同步也会对照上次游标对账删除,避免 `sync_mode=full` 漏删。目录遍历不完整时暂缓删除。 - **正文**:公开 Blocks API 只返回文档根下的一级块;高亮块等容器若响应里带有 `children` 会继续渲染,否则在元数据中标记 `nested_blocks_unavailable`,避免把残缺正文当成完整成功。 - **校验**:测试连接会列出知识库、探测根节点列表,并在根下存在在线文档时试读 Blocks;对已开启摄取的上传文件只解析存储位置并申请下载 URL,不下载正文,以便尽早发现缺少 `Wiki.Node.Read` / `Storage.File.Read`,同时让校验对多兆字节文件保持廉价。