feat(datasource/dingtalk): 保留来源目录结构(开关控制,默认关) (#4422)

main
linus-liu-web 2026-10-17 23:16:57 +08:00 committed by GitHub
parent 0aa1a517c8
commit e849eaad5c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 787 additions and 14 deletions

View File

@ -647,6 +647,12 @@ type syncScope struct {
Reference resourceReference
StartNodeID string
Document *node
// FolderPath is the folder hierarchy between the workspace root and the
// scope's own node, root-first, with the scope's own folder name last. It
// anchors every document the scope yields, so selecting a subfolder and
// selecting the whole workspace file the same document under the same
// folder instead of under two.
FolderPath []string
}
func (s syncScope) contains(candidate syncScope) bool {
@ -688,6 +694,9 @@ func (c *Connector) sync(
api := c.api(cfg)
settings := parseDocumentSettings(dataSourceConfig)
if settings.PreserveFolderStructure {
logger.Infof(ctx, "[DingTalk] source folder structure is encoded into document file names")
}
workspaces, err := api.listWorkspaces(ctx)
if err != nil {
return nil, nil, err
@ -696,6 +705,13 @@ func (c *Connector) sync(
if err != nil {
return nil, nil, err
}
// The workspace name is the first folder segment of every encoded path. It
// is read from the listing the sync already made, so the leading level
// costs no extra request.
workspaceFolderNames := make(map[string]string, len(workspaces))
for _, item := range workspaces {
workspaceFolderNames[item.ID] = workspaceFolderName(item)
}
next := &cursorState{
Version: cursorVersion,
@ -757,7 +773,8 @@ func (c *Connector) sync(
synced := 0
failed := 0
for _, document := range documents {
for _, scanned := range documents {
document := scanned.node
if document.ID == "" {
continue
}
@ -770,7 +787,13 @@ func (c *Connector) sync(
continue
}
item, err := readDocument(ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document)
folders := documentFolders(
settings, workspaceFolderNames[scope.Reference.WorkspaceID],
scope.FolderPath, scanned.folders,
)
item, err := readDocument(
ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document, folders,
)
if err != nil {
if isContextError(err) {
return nil, nil, err
@ -894,6 +917,7 @@ func resolveSyncScopes(
}, nil
}
var ancestorFolders []string
parentNodeID := rootNodeID
for _, ancestorID := range ref.Ancestors {
children, err := listChildren(parentNodeID)
@ -908,6 +932,11 @@ func resolveSyncScopes(
if ancestor.WorkspaceID != "" && ancestor.WorkspaceID != ref.WorkspaceID {
return syncScope{}, fmt.Errorf("DingTalk ancestor %q belongs to a different workspace", ancestorID)
}
// The name is read off a listing this resolution has to make
// anyway, so anchoring the path at the workspace root costs no
// extra request. A selection saved without ancestors yields a
// shallower path rather than an error.
ancestorFolders = append(ancestorFolders, ancestor.title())
parentNodeID = ancestor.ID
}
children, err := listChildren(parentNodeID)
@ -926,11 +955,13 @@ func resolveSyncScopes(
case selectedNode.isFolder():
return syncScope{
ResourceID: canonicalID, Reference: ref, StartNodeID: selectedNode.ID,
FolderPath: appendFolder(ancestorFolders, selectedNode.title()),
}, nil
case selectedNode.isIngestible(settings):
document := selectedNode
return syncScope{
ResourceID: canonicalID, Reference: ref, Document: &document,
FolderPath: ancestorFolders,
}, nil
default:
return syncScope{}, fmt.Errorf("DingTalk node %q is not a supported online document or folder",
@ -990,6 +1021,26 @@ func childByID(children []node, nodeID string) (node, bool) {
return node{}, false
}
// scannedDocument pairs an ingestible node with the folder hierarchy the walk
// found it under, relative to the scan's root node. The hierarchy is carried
// alongside the node rather than recomputed later: the breadth-first walk is
// the only place that knows which parent listed the node.
type scannedDocument struct {
node
// folders holds the folder titles between the scan root and the document,
// root-first. A document sitting directly in the scan root has none.
folders []string
}
// appendFolder returns parent plus one folder title. The result is always a
// fresh slice: appending in place would let one sibling's name overwrite
// another's inside a shared backing array.
func appendFolder(parent []string, name string) []string {
folders := make([]string, 0, len(parent)+1)
folders = append(folders, parent...)
return append(folders, name)
}
// scanScope lists every ingestible document in one scope, together with the
// nodes it saw but cannot ingest. The skipped nodes are returned rather than
// discarded so the caller can report them: a full sync must never look clean
@ -999,26 +1050,34 @@ func scanScope(
api dingTalkAPI,
scope syncScope,
settings documentSettings,
) ([]node, []node, error) {
) ([]scannedDocument, []node, error) {
if scope.Document != nil {
return []node{*scope.Document}, nil, nil
// A single-document scope has no tree to walk; the scope's own
// FolderPath already carries the folders above it.
return []scannedDocument{{node: *scope.Document}}, nil, nil
}
return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID, settings)
}
// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal
// only; every file is either an ingestible document or a skip.
// only; every file is either an ingestible document or a skip. Each document is
// reported with the folder titles the walk descended through to reach it, so
// the caller can mirror the source hierarchy in the knowledge base.
func scanWorkspace(
ctx context.Context,
api dingTalkAPI,
workspaceID string,
rootNodeID string,
settings documentSettings,
) ([]node, []node, error) {
) ([]scannedDocument, []node, error) {
queue := []string{rootNodeID}
visitedParents := make(map[string]struct{})
seenNodes := make(map[string]struct{})
var documents []node
// foldersByParent maps a node ID to the folder titles between the scan root
// and that node, root-first. Only nodes the walk descends through get an
// entry; the scan root itself has none.
foldersByParent := map[string][]string{rootNodeID: nil}
var documents []scannedDocument
var skipped []node
for len(queue) > 0 {
@ -1052,7 +1111,10 @@ func scanWorkspace(
}
switch {
case child.isIngestible(settings):
documents = append(documents, child)
documents = append(documents, scannedDocument{
node: child,
folders: foldersByParent[parentID],
})
case child.isFolder():
// Containers hold no content of their own, so they are never
// reported as skipped.
@ -1060,6 +1122,7 @@ func scanWorkspace(
skipped = append(skipped, child)
}
if child.isFolder() || child.HasChildren {
foldersByParent[child.ID] = appendFolder(foldersByParent[parentID], child.title())
queue = append(queue, child.ID)
}
}
@ -1141,13 +1204,16 @@ func warnUnknownBlockTypes(ctx context.Context, document node, rendered renderRe
// readDocument turns one supported wiki node into a FetchedItem. Native adoc
// documents are rendered from their blocks, and uploaded files are transferred
// byte-for-byte so WeKnora's own parsers can read them.
// byte-for-byte so WeKnora's own parsers can read them. folders is the source
// folder hierarchy to encode in the item's FileName, or nil to keep the
// historical flat name.
func readDocument(
ctx context.Context,
api dingTalkAPI,
sourceResourceID string,
workspaceID string,
document node,
folders []string,
) (types.FetchedItem, error) {
switch {
case document.isOnlineDocument():
@ -1160,16 +1226,73 @@ func readDocument(
// both the request context and the node identity are known. Uploaded
// files take the default branch and have no rendering step to lose.
warnUnknownBlockTypes(ctx, document, rendered)
return fetchedDocument(sourceResourceID, workspaceID, document, rendered), nil
return fetchedDocument(sourceResourceID, workspaceID, document, rendered, folders), nil
default:
data, err := api.downloadDocument(ctx, document.ID)
if err != nil {
return types.FetchedItem{}, err
}
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data), nil
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data, folders), nil
}
}
// documentFolders assembles the folder segments a document's FileName encodes:
// the workspace, the folders between the workspace root and the scope's own
// node, then the folders the scan descended through.
//
// It returns nil when the switch is off, which is what keeps the emitted
// FileName byte-for-byte identical to previous releases for every data source
// that never turned it on.
func documentFolders(
settings documentSettings,
workspaceName string,
scopeFolders []string,
scanFolders []string,
) []string {
if !settings.PreserveFolderStructure {
return nil
}
folders := make([]string, 0, len(scopeFolders)+len(scanFolders)+1)
if workspaceName != "" {
folders = append(folders, workspaceName)
}
folders = append(folders, scopeFolders...)
return append(folders, scanFolders...)
}
// workspaceFolderName names a workspace the way the resource picker does: the
// workspace name, falling back to its ID, which is what an operator sees when a
// tenant exposes no name.
func workspaceFolderName(item workspace) string {
if name := strings.TrimSpace(item.Name); name != "" {
return name
}
return item.ID
}
// buildFolderFileName prepends the folder hierarchy to a document's file name.
//
// folder_path downstream is derived from FileName alone
// (SplitKnowledgeRelativePath → NormalizeKnowledgeFolderPath), so the hierarchy
// has to be encoded here as a relative path.
//
// Every level is sanitised on its own, as the Yuque connector's own folder
// encoding does, so a "/" inside a folder name cannot silently create an extra
// nesting level. Folder names the downstream normaliser rejects outright — ".",
// "..", an empty name — drop their level instead of failing the sync. An empty
// folder list, which is what the default settings produce, returns the file
// name unchanged.
func buildFolderFileName(folders []string, fileName string) string {
if len(folders) == 0 {
return fileName
}
segments := make([]string, 0, len(folders))
for _, folder := range folders {
segments = append(segments, datasource.SanitizeFileName(folder))
}
return strings.Join(segments, "/") + "/" + fileName
}
// documentMetadata describes a wiki node the same way for rendered and
// downloaded documents, so both stay searchable by their DingTalk identity.
func documentMetadata(workspaceID string, document node) map[string]string {
@ -1195,6 +1318,7 @@ func fetchedDocument(
workspaceID string,
document node,
rendered renderResult,
folders []string,
) types.FetchedItem {
metadata := documentMetadata(workspaceID, document)
if len(rendered.UnknownTypes) > 0 {
@ -1205,7 +1329,7 @@ func fetchedDocument(
Title: document.title(),
Content: []byte(rendered.Markdown),
ContentType: "text/markdown",
FileName: renderedDocumentFileName(document.title()),
FileName: buildFolderFileName(folders, renderedDocumentFileName(document.title())),
URL: documentURL(document),
UpdatedAt: document.modifiedAt(),
Metadata: metadata,
@ -1234,6 +1358,7 @@ func fetchedBinaryDocument(
workspaceID string,
document node,
data []byte,
folders []string,
) types.FetchedItem {
contentType, ok := document.binaryContentType()
if !ok {
@ -1244,7 +1369,7 @@ func fetchedBinaryDocument(
Title: document.title(),
Content: data,
ContentType: contentType,
FileName: binaryDocumentFileName(document),
FileName: buildFolderFileName(folders, binaryDocumentFileName(document)),
URL: documentURL(document),
UpdatedAt: document.modifiedAt(),
Metadata: documentMetadata(workspaceID, document),

View File

@ -0,0 +1,198 @@
package dingtalk
import (
"context"
"encoding/json"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// --- settings ---
// The folder switch is read through the same helper as the uploaded-files
// switch, and an absent, malformed or unexpected value has to leave the data
// source on the historical flat layout.
func TestParseDocumentSettingsReadsTheFolderSwitch(t *testing.T) {
for _, testCase := range []struct {
label string
settings map[string]interface{}
want bool
}{
{"absent key", map[string]interface{}{}, false},
{"null settings entry", map[string]interface{}{"preserve_folder_structure": nil}, false},
{"bool true", map[string]interface{}{"preserve_folder_structure": true}, true},
{"bool false", map[string]interface{}{"preserve_folder_structure": false}, false},
{"numeric string", map[string]interface{}{"preserve_folder_structure": "1"}, true},
{"padded on", map[string]interface{}{"preserve_folder_structure": " on "}, true},
{"padded OFF", map[string]interface{}{"preserve_folder_structure": " OFF "}, false},
{"unknown spelling", map[string]interface{}{"preserve_folder_structure": "maybe"}, false},
{"unexpected type", map[string]interface{}{"preserve_folder_structure": 1}, false},
{
"independent of the uploaded-files switch",
map[string]interface{}{"include_uploaded_files": true},
false,
},
} {
t.Run(testCase.label, func(t *testing.T) {
dataSourceConfig := testConfig("ws")
dataSourceConfig.Settings = testCase.settings
got := parseDocumentSettings(dataSourceConfig)
if got.PreserveFolderStructure != testCase.want {
t.Fatalf("PreserveFolderStructure = %t, want %t", got.PreserveFolderStructure, testCase.want)
}
if got.IncludeUploadedFiles != settingBool(testCase.settings["include_uploaded_files"], false) {
t.Fatalf("IncludeUploadedFiles = %t, the two switches are not independent",
got.IncludeUploadedFiles)
}
})
}
if got := parseDocumentSettings(nil); got.PreserveFolderStructure {
t.Fatal("parseDocumentSettings(nil).PreserveFolderStructure = true, want false")
}
}
// documentFolders is the single place the switch is honoured, so the nil it
// returns when the switch is off is what keeps an existing data source
// byte-for-byte unchanged.
func TestDocumentFoldersIsNilUnlessTheSwitchIsOn(t *testing.T) {
scope := []string{"天诛渗透攻防"}
scan := []string{"2024 年度"}
off := documentFolders(documentSettings{}, "工作区", scope, scan)
if off != nil {
t.Fatalf("documentFolders() with the switch off = %#v, want nil", off)
}
on := documentFolders(
documentSettings{PreserveFolderStructure: true}, "工作区", scope, scan,
)
want := []string{"工作区", "天诛渗透攻防", "2024 年度"}
if strings.Join(on, "|") != strings.Join(want, "|") {
t.Fatalf("documentFolders() = %#v, want %#v", on, want)
}
// A workspace with no name of its own contributes no segment rather than an
// empty level.
unnamed := documentFolders(documentSettings{PreserveFolderStructure: true}, "", scope, nil)
if strings.Join(unnamed, "|") != "天诛渗透攻防" {
t.Fatalf("documentFolders() without a workspace name = %#v", unnamed)
}
}
// --- file name assembly ---
// The folder path a synced document ends up in is derived downstream from
// FileName alone, so the encoding is only correct if that split recovers the
// hierarchy.
func TestBuildFolderFileNameSurvivesTheDownstreamFolderSplit(t *testing.T) {
for _, testCase := range []struct {
label string
folders []string
fileName string
wantFolder string
wantName string
}{
{
"no folders keeps the historical shape",
nil, "FAQ.md", "", "FAQ.md",
},
{
"workspace only",
[]string{"工作区"},
"FAQ.md", "工作区", "FAQ.md",
},
{
"workspace and nested folders",
[]string{"天穹安全能力中心", "天诛渗透攻防", "2024 年度"},
"2024 年度复盘 FAQ.md",
"天穹安全能力中心/天诛渗透攻防/2024 年度",
"2024 年度复盘 FAQ.md",
},
{
"a separator inside a name cannot add a level",
[]string{"安全/中心", "A/B"},
"A_B 手册.md",
"安全_中心/A_B",
"A_B 手册.md",
},
{
"a level the normaliser rejects is dropped, not fatal",
[]string{"工作区", ".", "子目录"},
"FAQ.md",
"工作区/子目录",
"FAQ.md",
},
} {
t.Run(testCase.label, func(t *testing.T) {
raw := buildFolderFileName(testCase.folders, testCase.fileName)
folder, name := types.SplitKnowledgeRelativePath(raw)
if folder != testCase.wantFolder || name != testCase.wantName {
t.Fatalf("split(%q) = (%q, %q), want (%q, %q)",
raw, folder, name, testCase.wantFolder, testCase.wantName)
}
})
}
}
func TestBuildFolderFileNameWithoutFoldersIsTheIdentity(t *testing.T) {
for _, name := range []string{"FAQ.md", "手册.docx", "untitled"} {
if got := buildFolderFileName(nil, name); got != name {
t.Fatalf("buildFolderFileName(nil, %q) = %q, want it unchanged", name, got)
}
}
}
// Sibling folders are recorded while the walk visits them, so a name must not
// be written into a slice another sibling is still holding.
func TestAppendFolderNeverSharesABackingArray(t *testing.T) {
parent := make([]string, 1, 8)
parent[0] = "工作区"
first := appendFolder(parent, "天诛渗透攻防")
second := appendFolder(parent, "天禹漏洞情报")
if first[len(first)-1] != "天诛渗透攻防" {
t.Fatalf("first sibling = %#v, its name was overwritten", first)
}
if second[len(second)-1] != "天禹漏洞情报" {
t.Fatalf("second sibling = %#v, its name was overwritten", second)
}
if len(parent) != 1 || parent[0] != "工作区" {
t.Fatalf("appendFolder() mutated its input: %#v", parent)
}
}
// --- cursor safety ---
// The folder dimension is an ingest-path concern only. If it reached the sync
// cursor, turning the switch on would change what the next incremental sync
// considers unchanged — and, worse, which documents it considers deleted.
func TestFolderPathsNeverReachTheSyncCursor(t *testing.T) {
_, off, err := testConnector(tenantFolderPathFixture()).FetchIncremental(
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,
)
if err != nil {
t.Fatalf("FetchIncremental() with the switch off error = %v", err)
}
_, on, err := testConnector(tenantFolderPathFixture()).FetchIncremental(
context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), nil,
)
if err != nil {
t.Fatalf("FetchIncremental() with the switch on error = %v", err)
}
encode := func(t *testing.T, cursor *types.SyncCursor) string {
t.Helper()
if cursor == nil {
t.Fatal("FetchIncremental() returned no cursor")
}
raw, err := json.Marshal(cursor.ConnectorCursor["resources"])
if err != nil {
t.Fatal(err)
}
return string(raw)
}
offCursor, onCursor := encode(t, off), encode(t, on)
if offCursor != onCursor {
t.Fatalf("the cursor depends on the folder switch:\noff: %s\non: %s", offCursor, onCursor)
}
if strings.Contains(onCursor, tenantWorkspaceName) {
t.Fatalf("the cursor carries folder names: %s", onCursor)
}
}

View File

@ -0,0 +1,435 @@
package dingtalk
import (
"context"
"encoding/json"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// The fixture below reproduces the shape of the tenant that reported the
// problem: one workspace whose root holds five folders, a document that sits
// directly at the root, both formats inside the folders (native adoc FAQ pages
// and uploaded docx/pptx manuals), an empty folder, and one extra nesting level
// so the depth is not uniform. It goes through the public fetch entry points
// only, so every assertion describes what an operator sees in the knowledge
// base rather than how the connector is wired internally.
//
// Node names carry no file extension: DingTalk reports the extension in its own
// field, exactly as the rest of this package's fixtures do.
const (
tenantWorkspaceID = "ws-tianqiong"
tenantWorkspaceName = "天穹安全能力中心-天系列使用FAQ"
tenantRootNodeID = "root-tianqiong"
// tenantModifiedAt is the millisecond revision DingTalk reports for the
// fixture's documents. It is what makes an incremental sync skip an
// untouched document, so the relocation case below is about the file name
// and not about a missing revision.
tenantModifiedAt = int64(1760000000000)
)
// tenantUploadedFilesConfig is the state of the reported tenant before this
// change: uploaded manuals are ingested, the folder-path switch does not exist
// yet, so the settings bag has no key for it.
func tenantUploadedFilesConfig(resources ...string) *types.DataSourceConfig {
cfg := testConfig(resources...)
cfg.Settings = map[string]interface{}{"include_uploaded_files": true}
return cfg
}
// tenantFolderPathTestConfig turns the folder-path switch on as well.
func tenantFolderPathTestConfig(resources ...string) *types.DataSourceConfig {
cfg := tenantUploadedFilesConfig(resources...)
cfg.Settings["preserve_folder_structure"] = true
return cfg
}
// tenantFolderPathFixture builds the reported tenant tree.
func tenantFolderPathFixture() *fakeAPI {
adoc := func(id, name string) node {
return node{
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "ALIDOC",
Extension: "adoc", Name: name, ModifiedTimestamp: tenantModifiedAt,
}
}
upload := func(id, name, extension string) node {
return node{
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "DOCUMENT",
Extension: extension, Name: name, ModifiedTimestamp: tenantModifiedAt,
}
}
folder := func(id, name string) node {
return node{
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FOLDER", Name: name, HasChildren: true,
}
}
return &fakeAPI{
workspaces: []workspace{{
ID: tenantWorkspaceID, RootNodeID: tenantRootNodeID, Name: tenantWorkspaceName,
}},
nodes: map[string][]node{
tenantRootNodeID: {
folder("f-easm", "EASM-外部攻击面管理模块"),
adoc("d-cloud-faq", "天穹云端安全能力中枢常见问题 FAQ"),
folder("f-tianxi", "天析应急分析"),
folder("f-tianyu", "天禹漏洞情报"),
folder("f-tianfu", "天伏攻防知识库"),
folder("f-tianzhu", "天诛渗透攻防"),
},
"f-easm": {
adoc("d-easm-faq", "EASM 模块使用 FAQ"),
upload("d-easm-manual", "EASM 模块操作手册.docx", "docx"),
},
"f-tianxi": {
adoc("d-tianxi-faq", "天析应急分析 FAQ"),
upload("d-tianxi-deck", "天析应急分析介绍.pptx", "pptx"),
},
"f-tianyu": {
upload("d-tianyu-manual", "天禹漏洞情报使用手册.docx", "docx"),
upload("d-tianyu-deck", "天禹漏洞情报介绍.pptx", "pptx"),
},
// 天伏攻防知识库 is listed but holds nothing: an empty folder must
// stay invisible in the knowledge base instead of producing a level
// with no documents in it.
"f-tianfu": {},
"f-tianzhu": {
upload("d-tianzhu-manual", "天诛渗透攻防手册.docx", "docx"),
adoc("d-tianzhu-faq-1", "天诛渗透攻防 FAQ 一"),
adoc("d-tianzhu-faq-2", "天诛渗透攻防 FAQ 二"),
folder("f-tianzhu-2024", "2024 年度"),
},
"f-tianzhu-2024": {
adoc("d-tianzhu-2024-faq", "2024 年度复盘 FAQ"),
},
},
blocks: map[string][]json.RawMessage{
"d-cloud-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"cloud"}}`)},
"d-easm-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"easm"}}`)},
"d-tianxi-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"tianxi"}}`)},
"d-tianzhu-faq-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"one"}}`)},
"d-tianzhu-faq-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"two"}}`)},
"d-tianzhu-2024-faq": {
rawJSON(`{"blockType":"paragraph","paragraph":{"text":"review"}}`),
},
},
downloads: map[string][]byte{
"d-easm-manual": []byte("easm"),
"d-tianxi-deck": []byte("tianxi"),
"d-tianyu-manual": []byte("tianyu-manual"),
"d-tianyu-deck": []byte("tianyu-deck"),
"d-tianzhu-manual": []byte("tianzhu"),
},
nodeErrors: make(map[string]error),
blockErrors: make(map[string]error),
}
}
// fileNamesByDocument indexes the fetched items by node ID so an assertion can
// name the document it is talking about instead of an index.
func fileNamesByDocument(t *testing.T, items []types.FetchedItem) map[string]string {
t.Helper()
names := make(map[string]string, len(items))
for _, item := range items {
if _, seen := names[item.ExternalID]; seen {
t.Fatalf("document %q was fetched more than once", item.ExternalID)
}
names[item.ExternalID] = item.FileName
}
return names
}
func assertFileNames(t *testing.T, got map[string]string, want map[string]string) {
t.Helper()
for id, wantName := range want {
if got[id] != wantName {
t.Errorf("document %s FileName = %q, want %q", id, got[id], wantName)
}
}
if len(got) != len(want) {
t.Fatalf("fetched %d documents, want %d: %#v", len(got), len(want), got)
}
}
// Regression guard: a data source that never turned the switch on keeps the
// historical flat file names, byte for byte. Every document of the reported
// tenant is listed, in the same format it was ingested with before.
func TestDingTalkFolderPathsDefaultOffKeepsHistoricalFileNames(t *testing.T) {
captureLogs(t)
connector := testConnector(tenantFolderPathFixture())
items, err := connector.FetchAll(
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID),
[]string{tenantWorkspaceID},
)
if err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
"d-easm-faq": "EASM 模块使用 FAQ.md",
"d-easm-manual": "EASM 模块操作手册.docx",
"d-cloud-faq": "天穹云端安全能力中枢常见问题 FAQ.md",
"d-tianxi-faq": "天析应急分析 FAQ.md",
"d-tianxi-deck": "天析应急分析介绍.pptx",
"d-tianyu-manual": "天禹漏洞情报使用手册.docx",
"d-tianyu-deck": "天禹漏洞情报介绍.pptx",
"d-tianzhu-manual": "天诛渗透攻防手册.docx",
"d-tianzhu-faq-1": "天诛渗透攻防 FAQ 一.md",
"d-tianzhu-faq-2": "天诛渗透攻防 FAQ 二.md",
"d-tianzhu-2024-faq": "2024 年度复盘 FAQ.md",
})
}
// The reported problem, end to end: with the switch on, every document encodes
// its source folder hierarchy relative to the workspace root, so the five
// top-level folders stay distinguishable in the knowledge base. The workspace
// name is the first segment, and a document sitting at the root carries no
// folder segment beyond it.
func TestDingTalkFolderPathsPreserveTheTenantShapeWhenEnabled(t *testing.T) {
captureLogs(t)
connector := testConnector(tenantFolderPathFixture())
items, err := connector.FetchAll(
context.Background(),
tenantFolderPathTestConfig(tenantWorkspaceID),
[]string{tenantWorkspaceID},
)
if err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
root := tenantWorkspaceName + "/"
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
"d-easm-faq": root + "EASM-外部攻击面管理模块/EASM 模块使用 FAQ.md",
"d-easm-manual": root + "EASM-外部攻击面管理模块/EASM 模块操作手册.docx",
"d-cloud-faq": root + "天穹云端安全能力中枢常见问题 FAQ.md",
"d-tianxi-faq": root + "天析应急分析/天析应急分析 FAQ.md",
"d-tianxi-deck": root + "天析应急分析/天析应急分析介绍.pptx",
"d-tianyu-manual": root + "天禹漏洞情报/天禹漏洞情报使用手册.docx",
"d-tianyu-deck": root + "天禹漏洞情报/天禹漏洞情报介绍.pptx",
"d-tianzhu-manual": root + "天诛渗透攻防/天诛渗透攻防手册.docx",
"d-tianzhu-faq-1": root + "天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
"d-tianzhu-faq-2": root + "天诛渗透攻防/天诛渗透攻防 FAQ 二.md",
"d-tianzhu-2024-faq": root + "天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md",
})
}
// The switch has to read the same string spellings the connector's other switch
// reads, and anything unrecognised — including an absent key — has to leave the
// historical behaviour alone.
func TestDingTalkFolderPathsReadsTheSameSettingsSpellings(t *testing.T) {
captureLogs(t)
for _, testCase := range []struct {
label string
settings map[string]interface{}
want string
}{
{
"key absent",
map[string]interface{}{"include_uploaded_files": true},
"天诛渗透攻防 FAQ 一.md",
},
{
"bool true",
map[string]interface{}{"preserve_folder_structure": true},
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
},
{
"string yes",
map[string]interface{}{"preserve_folder_structure": "yes"},
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
},
{
"string ON with padding",
map[string]interface{}{"preserve_folder_structure": " ON "},
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
},
{
"bool false",
map[string]interface{}{"preserve_folder_structure": false},
"天诛渗透攻防 FAQ 一.md",
},
{
"unknown spelling",
map[string]interface{}{"preserve_folder_structure": "sometimes"},
"天诛渗透攻防 FAQ 一.md",
},
{
"unexpected type",
map[string]interface{}{"preserve_folder_structure": 42},
"天诛渗透攻防 FAQ 一.md",
},
} {
t.Run(testCase.label, func(t *testing.T) {
cfg := tenantUploadedFilesConfig(tenantWorkspaceID)
cfg.Settings = testCase.settings
connector := testConnector(tenantFolderPathFixture())
items, err := connector.FetchAll(
context.Background(), cfg, []string{tenantWorkspaceID},
)
if err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
names := fileNamesByDocument(t, items)
if names["d-tianzhu-faq-1"] != testCase.want {
t.Fatalf("FileName = %q, want %q", names["d-tianzhu-faq-1"], testCase.want)
}
})
}
}
// Selecting a folder is the common case, and the path a document lands on must
// not depend on how deep the operator drilled into the picker: the same
// document selected through the whole workspace, through its own folder and as
// a single document has to end up in one folder, not three.
func TestDingTalkFolderPathsDoNotDependOnTheSelectedScope(t *testing.T) {
captureLogs(t)
connector := testConnector(tenantFolderPathFixture())
wholeWorkspace, err := connector.FetchAll(
context.Background(),
tenantFolderPathTestConfig(tenantWorkspaceID),
[]string{tenantWorkspaceID},
)
if err != nil {
t.Fatalf("workspace scope error = %v", err)
}
folderID, err := encodeResourceReference(resourceReference{
WorkspaceID: tenantWorkspaceID, NodeID: "f-tianzhu-2024", Ancestors: []string{"f-tianzhu"},
})
if err != nil {
t.Fatal(err)
}
nestedFolder, err := connector.FetchAll(
context.Background(),
tenantFolderPathTestConfig(folderID),
[]string{folderID},
)
if err != nil {
t.Fatalf("folder scope error = %v", err)
}
documentID, err := encodeResourceReference(resourceReference{
WorkspaceID: tenantWorkspaceID, NodeID: "d-tianzhu-2024-faq",
Ancestors: []string{"f-tianzhu", "f-tianzhu-2024"},
})
if err != nil {
t.Fatal(err)
}
singleDocument, err := connector.FetchAll(
context.Background(),
tenantFolderPathTestConfig(documentID),
[]string{documentID},
)
if err != nil {
t.Fatalf("document scope error = %v", err)
}
want := tenantWorkspaceName + "/天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md"
for label, items := range map[string][]types.FetchedItem{
"workspace": wholeWorkspace,
"folder": nestedFolder,
"document": singleDocument,
} {
if got := fileNamesByDocument(t, items)["d-tianzhu-2024-faq"]; got != want {
t.Errorf("%s scope FileName = %q, want %q", label, got, want)
}
}
}
// Names are hostile but not tidy: a folder name may contain the path separator,
// punctuation DingTalk allows and file systems do not, and two sibling folders
// may share a name. Each level is sanitised on its own so a "/" inside one name
// cannot invent an extra level, and same-named siblings merge into one folder —
// the same trade-off the Yuque connector documents for same-named groups.
func TestDingTalkFolderPathsSanitizeEachLevelAndMergeSameNamedFolders(t *testing.T) {
captureLogs(t)
const workspaceName = "安全/中心"
adoc := func(id, name string) node {
return node{
ID: id, WorkspaceID: "ws-odd", Type: "FILE", Category: "ALIDOC",
Extension: "adoc", Name: name,
}
}
folder := func(id, name string) node {
return node{
ID: id, WorkspaceID: "ws-odd", Type: "FOLDER", Name: name, HasChildren: true,
}
}
api := &fakeAPI{
workspaces: []workspace{{ID: "ws-odd", RootNodeID: "root-odd", Name: workspaceName}},
nodes: map[string][]node{
"root-odd": {
folder("f-slash", "A/B"),
folder("f-punct", `<>:"|?*`),
// Two folders with the same title: DingTalk node IDs are
// unique, titles are not.
folder("f-dup-1", "重复"),
folder("f-dup-2", "重复"),
// A name of only whitespace falls back to the node ID, which
// is what the resource picker shows for it too.
folder("f-unnamed", " "),
},
"f-slash": {adoc("d-slash", "A/B 手册")},
"f-punct": {adoc("d-punct", "符号")},
"f-dup-1": {adoc("d-dup-1", "第一份")},
"f-dup-2": {adoc("d-dup-2", "第二份")},
"f-unnamed": {adoc("d-unnamed", "无名目录")},
},
blocks: map[string][]json.RawMessage{
"d-slash": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"a"}}`)},
"d-punct": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"b"}}`)},
"d-dup-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"c"}}`)},
"d-dup-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"d"}}`)},
"d-unnamed": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"e"}}`)},
},
nodeErrors: make(map[string]error),
blockErrors: make(map[string]error),
}
connector := testConnector(api)
items, err := connector.FetchAll(
context.Background(), tenantFolderPathTestConfig("ws-odd"), []string{"ws-odd"},
)
if err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
// "安全/中心" cannot invent a level either, and neither can "A/B" or the
// document name: the file name keeps the sanitised form it already had.
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
"d-slash": "安全_中心/A_B/A_B 手册.md",
"d-punct": "安全_中心/_______/符号.md",
"d-dup-1": "安全_中心/重复/第一份.md",
"d-dup-2": "安全_中心/重复/第二份.md",
"d-unnamed": "安全_中心/f-unnamed/无名目录.md",
})
}
// Turning the switch on in an existing data source is an ingest-path change,
// not a migration: an incremental sync compares content revisions, so the
// documents already in the knowledge base are not re-read and therefore keep
// the flat file names they were ingested with — and, above all, the folder
// dimension must never make the connector report them as deleted.
func TestDingTalkEnablingFolderPathsDoesNotRelocateOrDeleteExistingDocuments(t *testing.T) {
captureLogs(t)
connector := testConnector(tenantFolderPathFixture())
items, cursor, err := connector.FetchIncremental(
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,
)
if err != nil {
t.Fatalf("first FetchIncremental() error = %v", err)
}
if names := fileNamesByDocument(t, items); names["d-tianzhu-faq-1"] != "天诛渗透攻防 FAQ 一.md" {
t.Fatalf("first sync FileName = %q, want the flat name", names["d-tianzhu-faq-1"])
}
items, _, err = connector.FetchIncremental(
context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), cursor,
)
if err != nil {
t.Fatalf("second FetchIncremental() error = %v", err)
}
for _, item := range items {
if item.IsDeleted {
t.Fatalf("enabling folder paths reported document %q as deleted", item.ExternalID)
}
}
if len(items) != 0 {
t.Fatalf("second sync returned %d items, want none: unchanged documents are not "+
"re-read, so they keep the file name they were ingested with", len(items))
}
}

View File

@ -37,6 +37,14 @@ type documentSettings struct {
// IncludeUploadedFiles admits uploaded Office/PDF files (FILE/DOCUMENT
// nodes) that are downloaded from 钉盘 and parsed by WeKnora itself.
IncludeUploadedFiles bool
// PreserveFolderStructure encodes each document's DingTalk folder hierarchy
// into its FileName, so the document lands in the knowledge base folder
// that mirrors where it lives in the wiki instead of at the root.
//
// It is an ingest-path switch, not a migration. An already-ingested
// document is only re-read when its content changes, so turning the switch
// on does not relocate it; a full sync does.
PreserveFolderStructure bool
}
// parseDocumentSettings reads the settings bag of a data source. A missing or
@ -48,6 +56,7 @@ func parseDocumentSettings(dataSourceConfig *types.DataSourceConfig) documentSet
raw = dataSourceConfig.Settings
}
return documentSettings{
IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),
IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),
PreserveFolderStructure: settingBool(raw["preserve_folder_structure"], false),
}
}

View File

@ -233,6 +233,12 @@
- **上传文件(默认关闭)**:`Settings.include_uploaded_files`(默认 `false`)开启后,所选范围内钉盘中的普通上传附件(`FILE/DOCUMENT` 的 `docx` / `pptx` / `xlsx` / `pdf`)会被下载并交给 WeKnora 自带解析器入库,文件名保留原扩展名、按扩展名选择解析器;单个文件上限 64 MiB,超出上限即失败且不重试(重签名只会拿到同样的字节)。下载失败(401/403/404 等)会重新申请签名后重试,最多 3 次。
- **开关的默认值与成本**:`include_uploaded_files` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后的下一次同步(含增量同步)行为不变**,不会突然开始下载和向量化附件。打开后,这些文件的正文会进入知识库,占用存储并产生 embedding 成本;不需要时请保持关闭。关闭时连接校验、资源列表和同步都会跳过上传附件,同步日志给出的原因是 `include_uploaded_files` 未启用,而不是类型不支持。
- **开关的作用点**:测试连接(校验)、资源选择列表、全量同步与增量同步共用同一个判据,一处关闭即处处一致。
- **目录结构(默认关闭)**:钉钉知识库本身就是目录树,但连接器历来把文档平铺到知识库根目录,`folder_path` 为空,不同来源目录的文档在知识库里无法区分。`Settings.preserve_folder_structure`(默认 `false`)开启后,`FileName` 编码为 `<工作区名>/<子目录>/…/<标题>.ext`,由下游 `SplitKnowledgeRelativePath` → `NormalizeKnowledgeFolderPath` 拆出 `folder_path`——与语雀 `folder_mode=toc`、飞书云盘目录映射是同一套机制。**工作区名是第一段**:一个知识库可以同时链接多个钉钉数据源,没有这一段,不同工作区里同名的顶层目录会静默合并成一个(语雀用书名作第一段是同样的理由);工作区没有名字时退回用它的 ID。**路径始终以工作区为根**:选整个工作区、选某个子目录、选单篇文档,同一篇文档都落到同一个目录,不会出现两份。每一层目录名单独清洗(名字里的 `/`、`\`、`:` 等替换为 `_`),因此目录名里的斜杠不会凭空多出一层;同名子目录合并为同一个知识库目录(钉钉节点 ID 唯一、标题不唯一),`.`、`..` 这类非法层级由下游归一化丢弃而不是让同步失败。开启后目录由连接器托管:在知识库里手动调整过的目录,会在该文档下次同步时被覆盖回钉钉的结构。
- **目录开关的默认值与作用范围**:`preserve_folder_structure` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后 `FileName` 逐字节不变、`folder_path` 仍为空**。它只决定**入库路径**,不决定哪些文档入库:测试连接与资源选择列表不受它影响,关掉它也不会让任何文档消失。目录维度不写入同步游标——开关开与关时游标里的"资源 → 文档版本"表完全一致,所以目录永远不会让文档被判定为"源端已删除"。
- **目录相关注意 / 限制**:
- **开启开关不会自动归位已导入的文档**:增量同步按 `modifiedTimestamp` 判断变化,只重新读取内容有变化的文档,因此**已经在知识库里的文档会保持原来的扁平文件名**——它们并没有被重新读取。只有内容变更过的文档、或手动触发一次全量同步之后,才会整体移到新目录。这与语雀修改 `folder_mode` 后的行为一致。
- **`include_uploaded_files` 关闭时**,上传附件(`docx` / `pptx` / `xlsx` / `pdf`)不入库,其所在目录自然也不会出现;在线 `adoc` 文档不受该开关影响。
- **目录深度与长度由下游统一裁剪**:`folder_path` 最多 16 层、单个目录名最多 128 字节、整条路径最多 1024 字节,超出部分被截断(与文件夹上传同一套规则)。
- **同步**:按文档 `modifiedTimestamp`(毫秒)增量读取,缺失时回退 `modifiedTime`;合并重叠选择。全量同步也会对照上次游标对账删除,避免 `sync_mode=full` 漏删。目录遍历不完整时暂缓删除。
- **正文**:公开 Blocks API 只返回文档根下的一级块;高亮块等容器若响应里带有 `children` 会继续渲染,否则在元数据中标记 `nested_blocks_unavailable`,避免把残缺正文当成完整成功。
- **校验**:测试连接会列出知识库、探测根节点列表,并在根下存在在线文档时试读 Blocks;对已开启摄取的上传文件只解析存储位置并申请下载 URL,不下载正文,以便尽早发现缺少 `Wiki.Node.Read` / `Storage.File.Read`,同时让校验对多兆字节文件保持廉价。