feat(datasource/dingtalk): 保留来源目录结构(开关控制,默认关) (#4422)
parent
0aa1a517c8
commit
e849eaad5c
|
|
@ -647,6 +647,12 @@ type syncScope struct {
|
|||
Reference resourceReference
|
||||
StartNodeID string
|
||||
Document *node
|
||||
// FolderPath is the folder hierarchy between the workspace root and the
|
||||
// scope's own node, root-first, with the scope's own folder name last. It
|
||||
// anchors every document the scope yields, so selecting a subfolder and
|
||||
// selecting the whole workspace file the same document under the same
|
||||
// folder instead of under two.
|
||||
FolderPath []string
|
||||
}
|
||||
|
||||
func (s syncScope) contains(candidate syncScope) bool {
|
||||
|
|
@ -688,6 +694,9 @@ func (c *Connector) sync(
|
|||
|
||||
api := c.api(cfg)
|
||||
settings := parseDocumentSettings(dataSourceConfig)
|
||||
if settings.PreserveFolderStructure {
|
||||
logger.Infof(ctx, "[DingTalk] source folder structure is encoded into document file names")
|
||||
}
|
||||
workspaces, err := api.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
|
@ -696,6 +705,13 @@ func (c *Connector) sync(
|
|||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// The workspace name is the first folder segment of every encoded path. It
|
||||
// is read from the listing the sync already made, so the leading level
|
||||
// costs no extra request.
|
||||
workspaceFolderNames := make(map[string]string, len(workspaces))
|
||||
for _, item := range workspaces {
|
||||
workspaceFolderNames[item.ID] = workspaceFolderName(item)
|
||||
}
|
||||
|
||||
next := &cursorState{
|
||||
Version: cursorVersion,
|
||||
|
|
@ -757,7 +773,8 @@ func (c *Connector) sync(
|
|||
synced := 0
|
||||
failed := 0
|
||||
|
||||
for _, document := range documents {
|
||||
for _, scanned := range documents {
|
||||
document := scanned.node
|
||||
if document.ID == "" {
|
||||
continue
|
||||
}
|
||||
|
|
@ -770,7 +787,13 @@ func (c *Connector) sync(
|
|||
continue
|
||||
}
|
||||
|
||||
item, err := readDocument(ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document)
|
||||
folders := documentFolders(
|
||||
settings, workspaceFolderNames[scope.Reference.WorkspaceID],
|
||||
scope.FolderPath, scanned.folders,
|
||||
)
|
||||
item, err := readDocument(
|
||||
ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document, folders,
|
||||
)
|
||||
if err != nil {
|
||||
if isContextError(err) {
|
||||
return nil, nil, err
|
||||
|
|
@ -894,6 +917,7 @@ func resolveSyncScopes(
|
|||
}, nil
|
||||
}
|
||||
|
||||
var ancestorFolders []string
|
||||
parentNodeID := rootNodeID
|
||||
for _, ancestorID := range ref.Ancestors {
|
||||
children, err := listChildren(parentNodeID)
|
||||
|
|
@ -908,6 +932,11 @@ func resolveSyncScopes(
|
|||
if ancestor.WorkspaceID != "" && ancestor.WorkspaceID != ref.WorkspaceID {
|
||||
return syncScope{}, fmt.Errorf("DingTalk ancestor %q belongs to a different workspace", ancestorID)
|
||||
}
|
||||
// The name is read off a listing this resolution has to make
|
||||
// anyway, so anchoring the path at the workspace root costs no
|
||||
// extra request. A selection saved without ancestors yields a
|
||||
// shallower path rather than an error.
|
||||
ancestorFolders = append(ancestorFolders, ancestor.title())
|
||||
parentNodeID = ancestor.ID
|
||||
}
|
||||
children, err := listChildren(parentNodeID)
|
||||
|
|
@ -926,11 +955,13 @@ func resolveSyncScopes(
|
|||
case selectedNode.isFolder():
|
||||
return syncScope{
|
||||
ResourceID: canonicalID, Reference: ref, StartNodeID: selectedNode.ID,
|
||||
FolderPath: appendFolder(ancestorFolders, selectedNode.title()),
|
||||
}, nil
|
||||
case selectedNode.isIngestible(settings):
|
||||
document := selectedNode
|
||||
return syncScope{
|
||||
ResourceID: canonicalID, Reference: ref, Document: &document,
|
||||
FolderPath: ancestorFolders,
|
||||
}, nil
|
||||
default:
|
||||
return syncScope{}, fmt.Errorf("DingTalk node %q is not a supported online document or folder",
|
||||
|
|
@ -990,6 +1021,26 @@ func childByID(children []node, nodeID string) (node, bool) {
|
|||
return node{}, false
|
||||
}
|
||||
|
||||
// scannedDocument pairs an ingestible node with the folder hierarchy the walk
|
||||
// found it under, relative to the scan's root node. The hierarchy is carried
|
||||
// alongside the node rather than recomputed later: the breadth-first walk is
|
||||
// the only place that knows which parent listed the node.
|
||||
type scannedDocument struct {
|
||||
node
|
||||
// folders holds the folder titles between the scan root and the document,
|
||||
// root-first. A document sitting directly in the scan root has none.
|
||||
folders []string
|
||||
}
|
||||
|
||||
// appendFolder returns parent plus one folder title. The result is always a
|
||||
// fresh slice: appending in place would let one sibling's name overwrite
|
||||
// another's inside a shared backing array.
|
||||
func appendFolder(parent []string, name string) []string {
|
||||
folders := make([]string, 0, len(parent)+1)
|
||||
folders = append(folders, parent...)
|
||||
return append(folders, name)
|
||||
}
|
||||
|
||||
// scanScope lists every ingestible document in one scope, together with the
|
||||
// nodes it saw but cannot ingest. The skipped nodes are returned rather than
|
||||
// discarded so the caller can report them: a full sync must never look clean
|
||||
|
|
@ -999,26 +1050,34 @@ func scanScope(
|
|||
api dingTalkAPI,
|
||||
scope syncScope,
|
||||
settings documentSettings,
|
||||
) ([]node, []node, error) {
|
||||
) ([]scannedDocument, []node, error) {
|
||||
if scope.Document != nil {
|
||||
return []node{*scope.Document}, nil, nil
|
||||
// A single-document scope has no tree to walk; the scope's own
|
||||
// FolderPath already carries the folders above it.
|
||||
return []scannedDocument{{node: *scope.Document}}, nil, nil
|
||||
}
|
||||
return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID, settings)
|
||||
}
|
||||
|
||||
// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal
|
||||
// only; every file is either an ingestible document or a skip.
|
||||
// only; every file is either an ingestible document or a skip. Each document is
|
||||
// reported with the folder titles the walk descended through to reach it, so
|
||||
// the caller can mirror the source hierarchy in the knowledge base.
|
||||
func scanWorkspace(
|
||||
ctx context.Context,
|
||||
api dingTalkAPI,
|
||||
workspaceID string,
|
||||
rootNodeID string,
|
||||
settings documentSettings,
|
||||
) ([]node, []node, error) {
|
||||
) ([]scannedDocument, []node, error) {
|
||||
queue := []string{rootNodeID}
|
||||
visitedParents := make(map[string]struct{})
|
||||
seenNodes := make(map[string]struct{})
|
||||
var documents []node
|
||||
// foldersByParent maps a node ID to the folder titles between the scan root
|
||||
// and that node, root-first. Only nodes the walk descends through get an
|
||||
// entry; the scan root itself has none.
|
||||
foldersByParent := map[string][]string{rootNodeID: nil}
|
||||
var documents []scannedDocument
|
||||
var skipped []node
|
||||
|
||||
for len(queue) > 0 {
|
||||
|
|
@ -1052,7 +1111,10 @@ func scanWorkspace(
|
|||
}
|
||||
switch {
|
||||
case child.isIngestible(settings):
|
||||
documents = append(documents, child)
|
||||
documents = append(documents, scannedDocument{
|
||||
node: child,
|
||||
folders: foldersByParent[parentID],
|
||||
})
|
||||
case child.isFolder():
|
||||
// Containers hold no content of their own, so they are never
|
||||
// reported as skipped.
|
||||
|
|
@ -1060,6 +1122,7 @@ func scanWorkspace(
|
|||
skipped = append(skipped, child)
|
||||
}
|
||||
if child.isFolder() || child.HasChildren {
|
||||
foldersByParent[child.ID] = appendFolder(foldersByParent[parentID], child.title())
|
||||
queue = append(queue, child.ID)
|
||||
}
|
||||
}
|
||||
|
|
@ -1141,13 +1204,16 @@ func warnUnknownBlockTypes(ctx context.Context, document node, rendered renderRe
|
|||
|
||||
// readDocument turns one supported wiki node into a FetchedItem. Native adoc
|
||||
// documents are rendered from their blocks, and uploaded files are transferred
|
||||
// byte-for-byte so WeKnora's own parsers can read them.
|
||||
// byte-for-byte so WeKnora's own parsers can read them. folders is the source
|
||||
// folder hierarchy to encode in the item's FileName, or nil to keep the
|
||||
// historical flat name.
|
||||
func readDocument(
|
||||
ctx context.Context,
|
||||
api dingTalkAPI,
|
||||
sourceResourceID string,
|
||||
workspaceID string,
|
||||
document node,
|
||||
folders []string,
|
||||
) (types.FetchedItem, error) {
|
||||
switch {
|
||||
case document.isOnlineDocument():
|
||||
|
|
@ -1160,16 +1226,73 @@ func readDocument(
|
|||
// both the request context and the node identity are known. Uploaded
|
||||
// files take the default branch and have no rendering step to lose.
|
||||
warnUnknownBlockTypes(ctx, document, rendered)
|
||||
return fetchedDocument(sourceResourceID, workspaceID, document, rendered), nil
|
||||
return fetchedDocument(sourceResourceID, workspaceID, document, rendered, folders), nil
|
||||
default:
|
||||
data, err := api.downloadDocument(ctx, document.ID)
|
||||
if err != nil {
|
||||
return types.FetchedItem{}, err
|
||||
}
|
||||
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data), nil
|
||||
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data, folders), nil
|
||||
}
|
||||
}
|
||||
|
||||
// documentFolders assembles the folder segments a document's FileName encodes:
|
||||
// the workspace, the folders between the workspace root and the scope's own
|
||||
// node, then the folders the scan descended through.
|
||||
//
|
||||
// It returns nil when the switch is off, which is what keeps the emitted
|
||||
// FileName byte-for-byte identical to previous releases for every data source
|
||||
// that never turned it on.
|
||||
func documentFolders(
|
||||
settings documentSettings,
|
||||
workspaceName string,
|
||||
scopeFolders []string,
|
||||
scanFolders []string,
|
||||
) []string {
|
||||
if !settings.PreserveFolderStructure {
|
||||
return nil
|
||||
}
|
||||
folders := make([]string, 0, len(scopeFolders)+len(scanFolders)+1)
|
||||
if workspaceName != "" {
|
||||
folders = append(folders, workspaceName)
|
||||
}
|
||||
folders = append(folders, scopeFolders...)
|
||||
return append(folders, scanFolders...)
|
||||
}
|
||||
|
||||
// workspaceFolderName names a workspace the way the resource picker does: the
|
||||
// workspace name, falling back to its ID, which is what an operator sees when a
|
||||
// tenant exposes no name.
|
||||
func workspaceFolderName(item workspace) string {
|
||||
if name := strings.TrimSpace(item.Name); name != "" {
|
||||
return name
|
||||
}
|
||||
return item.ID
|
||||
}
|
||||
|
||||
// buildFolderFileName prepends the folder hierarchy to a document's file name.
|
||||
//
|
||||
// folder_path downstream is derived from FileName alone
|
||||
// (SplitKnowledgeRelativePath → NormalizeKnowledgeFolderPath), so the hierarchy
|
||||
// has to be encoded here as a relative path.
|
||||
//
|
||||
// Every level is sanitised on its own, as the Yuque connector's own folder
|
||||
// encoding does, so a "/" inside a folder name cannot silently create an extra
|
||||
// nesting level. Folder names the downstream normaliser rejects outright — ".",
|
||||
// "..", an empty name — drop their level instead of failing the sync. An empty
|
||||
// folder list, which is what the default settings produce, returns the file
|
||||
// name unchanged.
|
||||
func buildFolderFileName(folders []string, fileName string) string {
|
||||
if len(folders) == 0 {
|
||||
return fileName
|
||||
}
|
||||
segments := make([]string, 0, len(folders))
|
||||
for _, folder := range folders {
|
||||
segments = append(segments, datasource.SanitizeFileName(folder))
|
||||
}
|
||||
return strings.Join(segments, "/") + "/" + fileName
|
||||
}
|
||||
|
||||
// documentMetadata describes a wiki node the same way for rendered and
|
||||
// downloaded documents, so both stay searchable by their DingTalk identity.
|
||||
func documentMetadata(workspaceID string, document node) map[string]string {
|
||||
|
|
@ -1195,6 +1318,7 @@ func fetchedDocument(
|
|||
workspaceID string,
|
||||
document node,
|
||||
rendered renderResult,
|
||||
folders []string,
|
||||
) types.FetchedItem {
|
||||
metadata := documentMetadata(workspaceID, document)
|
||||
if len(rendered.UnknownTypes) > 0 {
|
||||
|
|
@ -1205,7 +1329,7 @@ func fetchedDocument(
|
|||
Title: document.title(),
|
||||
Content: []byte(rendered.Markdown),
|
||||
ContentType: "text/markdown",
|
||||
FileName: renderedDocumentFileName(document.title()),
|
||||
FileName: buildFolderFileName(folders, renderedDocumentFileName(document.title())),
|
||||
URL: documentURL(document),
|
||||
UpdatedAt: document.modifiedAt(),
|
||||
Metadata: metadata,
|
||||
|
|
@ -1234,6 +1358,7 @@ func fetchedBinaryDocument(
|
|||
workspaceID string,
|
||||
document node,
|
||||
data []byte,
|
||||
folders []string,
|
||||
) types.FetchedItem {
|
||||
contentType, ok := document.binaryContentType()
|
||||
if !ok {
|
||||
|
|
@ -1244,7 +1369,7 @@ func fetchedBinaryDocument(
|
|||
Title: document.title(),
|
||||
Content: data,
|
||||
ContentType: contentType,
|
||||
FileName: binaryDocumentFileName(document),
|
||||
FileName: buildFolderFileName(folders, binaryDocumentFileName(document)),
|
||||
URL: documentURL(document),
|
||||
UpdatedAt: document.modifiedAt(),
|
||||
Metadata: documentMetadata(workspaceID, document),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,198 @@
|
|||
package dingtalk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// --- settings ---
|
||||
|
||||
// The folder switch is read through the same helper as the uploaded-files
|
||||
// switch, and an absent, malformed or unexpected value has to leave the data
|
||||
// source on the historical flat layout.
|
||||
func TestParseDocumentSettingsReadsTheFolderSwitch(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
label string
|
||||
settings map[string]interface{}
|
||||
want bool
|
||||
}{
|
||||
{"absent key", map[string]interface{}{}, false},
|
||||
{"null settings entry", map[string]interface{}{"preserve_folder_structure": nil}, false},
|
||||
{"bool true", map[string]interface{}{"preserve_folder_structure": true}, true},
|
||||
{"bool false", map[string]interface{}{"preserve_folder_structure": false}, false},
|
||||
{"numeric string", map[string]interface{}{"preserve_folder_structure": "1"}, true},
|
||||
{"padded on", map[string]interface{}{"preserve_folder_structure": " on "}, true},
|
||||
{"padded OFF", map[string]interface{}{"preserve_folder_structure": " OFF "}, false},
|
||||
{"unknown spelling", map[string]interface{}{"preserve_folder_structure": "maybe"}, false},
|
||||
{"unexpected type", map[string]interface{}{"preserve_folder_structure": 1}, false},
|
||||
{
|
||||
"independent of the uploaded-files switch",
|
||||
map[string]interface{}{"include_uploaded_files": true},
|
||||
false,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
dataSourceConfig := testConfig("ws")
|
||||
dataSourceConfig.Settings = testCase.settings
|
||||
got := parseDocumentSettings(dataSourceConfig)
|
||||
if got.PreserveFolderStructure != testCase.want {
|
||||
t.Fatalf("PreserveFolderStructure = %t, want %t", got.PreserveFolderStructure, testCase.want)
|
||||
}
|
||||
if got.IncludeUploadedFiles != settingBool(testCase.settings["include_uploaded_files"], false) {
|
||||
t.Fatalf("IncludeUploadedFiles = %t, the two switches are not independent",
|
||||
got.IncludeUploadedFiles)
|
||||
}
|
||||
})
|
||||
}
|
||||
if got := parseDocumentSettings(nil); got.PreserveFolderStructure {
|
||||
t.Fatal("parseDocumentSettings(nil).PreserveFolderStructure = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// documentFolders is the single place the switch is honoured, so the nil it
|
||||
// returns when the switch is off is what keeps an existing data source
|
||||
// byte-for-byte unchanged.
|
||||
func TestDocumentFoldersIsNilUnlessTheSwitchIsOn(t *testing.T) {
|
||||
scope := []string{"天诛渗透攻防"}
|
||||
scan := []string{"2024 年度"}
|
||||
off := documentFolders(documentSettings{}, "工作区", scope, scan)
|
||||
if off != nil {
|
||||
t.Fatalf("documentFolders() with the switch off = %#v, want nil", off)
|
||||
}
|
||||
on := documentFolders(
|
||||
documentSettings{PreserveFolderStructure: true}, "工作区", scope, scan,
|
||||
)
|
||||
want := []string{"工作区", "天诛渗透攻防", "2024 年度"}
|
||||
if strings.Join(on, "|") != strings.Join(want, "|") {
|
||||
t.Fatalf("documentFolders() = %#v, want %#v", on, want)
|
||||
}
|
||||
// A workspace with no name of its own contributes no segment rather than an
|
||||
// empty level.
|
||||
unnamed := documentFolders(documentSettings{PreserveFolderStructure: true}, "", scope, nil)
|
||||
if strings.Join(unnamed, "|") != "天诛渗透攻防" {
|
||||
t.Fatalf("documentFolders() without a workspace name = %#v", unnamed)
|
||||
}
|
||||
}
|
||||
|
||||
// --- file name assembly ---
|
||||
|
||||
// The folder path a synced document ends up in is derived downstream from
|
||||
// FileName alone, so the encoding is only correct if that split recovers the
|
||||
// hierarchy.
|
||||
func TestBuildFolderFileNameSurvivesTheDownstreamFolderSplit(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
label string
|
||||
folders []string
|
||||
fileName string
|
||||
wantFolder string
|
||||
wantName string
|
||||
}{
|
||||
{
|
||||
"no folders keeps the historical shape",
|
||||
nil, "FAQ.md", "", "FAQ.md",
|
||||
},
|
||||
{
|
||||
"workspace only",
|
||||
[]string{"工作区"},
|
||||
"FAQ.md", "工作区", "FAQ.md",
|
||||
},
|
||||
{
|
||||
"workspace and nested folders",
|
||||
[]string{"天穹安全能力中心", "天诛渗透攻防", "2024 年度"},
|
||||
"2024 年度复盘 FAQ.md",
|
||||
"天穹安全能力中心/天诛渗透攻防/2024 年度",
|
||||
"2024 年度复盘 FAQ.md",
|
||||
},
|
||||
{
|
||||
"a separator inside a name cannot add a level",
|
||||
[]string{"安全/中心", "A/B"},
|
||||
"A_B 手册.md",
|
||||
"安全_中心/A_B",
|
||||
"A_B 手册.md",
|
||||
},
|
||||
{
|
||||
"a level the normaliser rejects is dropped, not fatal",
|
||||
[]string{"工作区", ".", "子目录"},
|
||||
"FAQ.md",
|
||||
"工作区/子目录",
|
||||
"FAQ.md",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
raw := buildFolderFileName(testCase.folders, testCase.fileName)
|
||||
folder, name := types.SplitKnowledgeRelativePath(raw)
|
||||
if folder != testCase.wantFolder || name != testCase.wantName {
|
||||
t.Fatalf("split(%q) = (%q, %q), want (%q, %q)",
|
||||
raw, folder, name, testCase.wantFolder, testCase.wantName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFolderFileNameWithoutFoldersIsTheIdentity(t *testing.T) {
|
||||
for _, name := range []string{"FAQ.md", "手册.docx", "untitled"} {
|
||||
if got := buildFolderFileName(nil, name); got != name {
|
||||
t.Fatalf("buildFolderFileName(nil, %q) = %q, want it unchanged", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sibling folders are recorded while the walk visits them, so a name must not
|
||||
// be written into a slice another sibling is still holding.
|
||||
func TestAppendFolderNeverSharesABackingArray(t *testing.T) {
|
||||
parent := make([]string, 1, 8)
|
||||
parent[0] = "工作区"
|
||||
first := appendFolder(parent, "天诛渗透攻防")
|
||||
second := appendFolder(parent, "天禹漏洞情报")
|
||||
if first[len(first)-1] != "天诛渗透攻防" {
|
||||
t.Fatalf("first sibling = %#v, its name was overwritten", first)
|
||||
}
|
||||
if second[len(second)-1] != "天禹漏洞情报" {
|
||||
t.Fatalf("second sibling = %#v, its name was overwritten", second)
|
||||
}
|
||||
if len(parent) != 1 || parent[0] != "工作区" {
|
||||
t.Fatalf("appendFolder() mutated its input: %#v", parent)
|
||||
}
|
||||
}
|
||||
|
||||
// --- cursor safety ---
|
||||
|
||||
// The folder dimension is an ingest-path concern only. If it reached the sync
|
||||
// cursor, turning the switch on would change what the next incremental sync
|
||||
// considers unchanged — and, worse, which documents it considers deleted.
|
||||
func TestFolderPathsNeverReachTheSyncCursor(t *testing.T) {
|
||||
_, off, err := testConnector(tenantFolderPathFixture()).FetchIncremental(
|
||||
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchIncremental() with the switch off error = %v", err)
|
||||
}
|
||||
_, on, err := testConnector(tenantFolderPathFixture()).FetchIncremental(
|
||||
context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchIncremental() with the switch on error = %v", err)
|
||||
}
|
||||
encode := func(t *testing.T, cursor *types.SyncCursor) string {
|
||||
t.Helper()
|
||||
if cursor == nil {
|
||||
t.Fatal("FetchIncremental() returned no cursor")
|
||||
}
|
||||
raw, err := json.Marshal(cursor.ConnectorCursor["resources"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
offCursor, onCursor := encode(t, off), encode(t, on)
|
||||
if offCursor != onCursor {
|
||||
t.Fatalf("the cursor depends on the folder switch:\noff: %s\non: %s", offCursor, onCursor)
|
||||
}
|
||||
if strings.Contains(onCursor, tenantWorkspaceName) {
|
||||
t.Fatalf("the cursor carries folder names: %s", onCursor)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,435 @@
|
|||
package dingtalk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// The fixture below reproduces the shape of the tenant that reported the
|
||||
// problem: one workspace whose root holds five folders, a document that sits
|
||||
// directly at the root, both formats inside the folders (native adoc FAQ pages
|
||||
// and uploaded docx/pptx manuals), an empty folder, and one extra nesting level
|
||||
// so the depth is not uniform. It goes through the public fetch entry points
|
||||
// only, so every assertion describes what an operator sees in the knowledge
|
||||
// base rather than how the connector is wired internally.
|
||||
//
|
||||
// Node names carry no file extension: DingTalk reports the extension in its own
|
||||
// field, exactly as the rest of this package's fixtures do.
|
||||
const (
|
||||
tenantWorkspaceID = "ws-tianqiong"
|
||||
tenantWorkspaceName = "天穹安全能力中心-天系列使用FAQ"
|
||||
tenantRootNodeID = "root-tianqiong"
|
||||
// tenantModifiedAt is the millisecond revision DingTalk reports for the
|
||||
// fixture's documents. It is what makes an incremental sync skip an
|
||||
// untouched document, so the relocation case below is about the file name
|
||||
// and not about a missing revision.
|
||||
tenantModifiedAt = int64(1760000000000)
|
||||
)
|
||||
|
||||
// tenantUploadedFilesConfig is the state of the reported tenant before this
|
||||
// change: uploaded manuals are ingested, the folder-path switch does not exist
|
||||
// yet, so the settings bag has no key for it.
|
||||
func tenantUploadedFilesConfig(resources ...string) *types.DataSourceConfig {
|
||||
cfg := testConfig(resources...)
|
||||
cfg.Settings = map[string]interface{}{"include_uploaded_files": true}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// tenantFolderPathTestConfig turns the folder-path switch on as well.
|
||||
func tenantFolderPathTestConfig(resources ...string) *types.DataSourceConfig {
|
||||
cfg := tenantUploadedFilesConfig(resources...)
|
||||
cfg.Settings["preserve_folder_structure"] = true
|
||||
return cfg
|
||||
}
|
||||
|
||||
// tenantFolderPathFixture builds the reported tenant tree.
|
||||
func tenantFolderPathFixture() *fakeAPI {
|
||||
adoc := func(id, name string) node {
|
||||
return node{
|
||||
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "ALIDOC",
|
||||
Extension: "adoc", Name: name, ModifiedTimestamp: tenantModifiedAt,
|
||||
}
|
||||
}
|
||||
upload := func(id, name, extension string) node {
|
||||
return node{
|
||||
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "DOCUMENT",
|
||||
Extension: extension, Name: name, ModifiedTimestamp: tenantModifiedAt,
|
||||
}
|
||||
}
|
||||
folder := func(id, name string) node {
|
||||
return node{
|
||||
ID: id, WorkspaceID: tenantWorkspaceID, Type: "FOLDER", Name: name, HasChildren: true,
|
||||
}
|
||||
}
|
||||
return &fakeAPI{
|
||||
workspaces: []workspace{{
|
||||
ID: tenantWorkspaceID, RootNodeID: tenantRootNodeID, Name: tenantWorkspaceName,
|
||||
}},
|
||||
nodes: map[string][]node{
|
||||
tenantRootNodeID: {
|
||||
folder("f-easm", "EASM-外部攻击面管理模块"),
|
||||
adoc("d-cloud-faq", "天穹云端安全能力中枢常见问题 FAQ"),
|
||||
folder("f-tianxi", "天析应急分析"),
|
||||
folder("f-tianyu", "天禹漏洞情报"),
|
||||
folder("f-tianfu", "天伏攻防知识库"),
|
||||
folder("f-tianzhu", "天诛渗透攻防"),
|
||||
},
|
||||
"f-easm": {
|
||||
adoc("d-easm-faq", "EASM 模块使用 FAQ"),
|
||||
upload("d-easm-manual", "EASM 模块操作手册.docx", "docx"),
|
||||
},
|
||||
"f-tianxi": {
|
||||
adoc("d-tianxi-faq", "天析应急分析 FAQ"),
|
||||
upload("d-tianxi-deck", "天析应急分析介绍.pptx", "pptx"),
|
||||
},
|
||||
"f-tianyu": {
|
||||
upload("d-tianyu-manual", "天禹漏洞情报使用手册.docx", "docx"),
|
||||
upload("d-tianyu-deck", "天禹漏洞情报介绍.pptx", "pptx"),
|
||||
},
|
||||
// 天伏攻防知识库 is listed but holds nothing: an empty folder must
|
||||
// stay invisible in the knowledge base instead of producing a level
|
||||
// with no documents in it.
|
||||
"f-tianfu": {},
|
||||
"f-tianzhu": {
|
||||
upload("d-tianzhu-manual", "天诛渗透攻防手册.docx", "docx"),
|
||||
adoc("d-tianzhu-faq-1", "天诛渗透攻防 FAQ 一"),
|
||||
adoc("d-tianzhu-faq-2", "天诛渗透攻防 FAQ 二"),
|
||||
folder("f-tianzhu-2024", "2024 年度"),
|
||||
},
|
||||
"f-tianzhu-2024": {
|
||||
adoc("d-tianzhu-2024-faq", "2024 年度复盘 FAQ"),
|
||||
},
|
||||
},
|
||||
blocks: map[string][]json.RawMessage{
|
||||
"d-cloud-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"cloud"}}`)},
|
||||
"d-easm-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"easm"}}`)},
|
||||
"d-tianxi-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"tianxi"}}`)},
|
||||
"d-tianzhu-faq-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"one"}}`)},
|
||||
"d-tianzhu-faq-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"two"}}`)},
|
||||
"d-tianzhu-2024-faq": {
|
||||
rawJSON(`{"blockType":"paragraph","paragraph":{"text":"review"}}`),
|
||||
},
|
||||
},
|
||||
downloads: map[string][]byte{
|
||||
"d-easm-manual": []byte("easm"),
|
||||
"d-tianxi-deck": []byte("tianxi"),
|
||||
"d-tianyu-manual": []byte("tianyu-manual"),
|
||||
"d-tianyu-deck": []byte("tianyu-deck"),
|
||||
"d-tianzhu-manual": []byte("tianzhu"),
|
||||
},
|
||||
nodeErrors: make(map[string]error),
|
||||
blockErrors: make(map[string]error),
|
||||
}
|
||||
}
|
||||
|
||||
// fileNamesByDocument indexes the fetched items by node ID so an assertion can
|
||||
// name the document it is talking about instead of an index.
|
||||
func fileNamesByDocument(t *testing.T, items []types.FetchedItem) map[string]string {
|
||||
t.Helper()
|
||||
names := make(map[string]string, len(items))
|
||||
for _, item := range items {
|
||||
if _, seen := names[item.ExternalID]; seen {
|
||||
t.Fatalf("document %q was fetched more than once", item.ExternalID)
|
||||
}
|
||||
names[item.ExternalID] = item.FileName
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func assertFileNames(t *testing.T, got map[string]string, want map[string]string) {
|
||||
t.Helper()
|
||||
for id, wantName := range want {
|
||||
if got[id] != wantName {
|
||||
t.Errorf("document %s FileName = %q, want %q", id, got[id], wantName)
|
||||
}
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("fetched %d documents, want %d: %#v", len(got), len(want), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression guard: a data source that never turned the switch on keeps the
|
||||
// historical flat file names, byte for byte. Every document of the reported
|
||||
// tenant is listed, in the same format it was ingested with before.
|
||||
func TestDingTalkFolderPathsDefaultOffKeepsHistoricalFileNames(t *testing.T) {
|
||||
captureLogs(t)
|
||||
connector := testConnector(tenantFolderPathFixture())
|
||||
items, err := connector.FetchAll(
|
||||
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID),
|
||||
[]string{tenantWorkspaceID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
|
||||
"d-easm-faq": "EASM 模块使用 FAQ.md",
|
||||
"d-easm-manual": "EASM 模块操作手册.docx",
|
||||
"d-cloud-faq": "天穹云端安全能力中枢常见问题 FAQ.md",
|
||||
"d-tianxi-faq": "天析应急分析 FAQ.md",
|
||||
"d-tianxi-deck": "天析应急分析介绍.pptx",
|
||||
"d-tianyu-manual": "天禹漏洞情报使用手册.docx",
|
||||
"d-tianyu-deck": "天禹漏洞情报介绍.pptx",
|
||||
"d-tianzhu-manual": "天诛渗透攻防手册.docx",
|
||||
"d-tianzhu-faq-1": "天诛渗透攻防 FAQ 一.md",
|
||||
"d-tianzhu-faq-2": "天诛渗透攻防 FAQ 二.md",
|
||||
"d-tianzhu-2024-faq": "2024 年度复盘 FAQ.md",
|
||||
})
|
||||
}
|
||||
|
||||
// The reported problem, end to end: with the switch on, every document encodes
|
||||
// its source folder hierarchy relative to the workspace root, so the five
|
||||
// top-level folders stay distinguishable in the knowledge base. The workspace
|
||||
// name is the first segment, and a document sitting at the root carries no
|
||||
// folder segment beyond it.
|
||||
func TestDingTalkFolderPathsPreserveTheTenantShapeWhenEnabled(t *testing.T) {
|
||||
captureLogs(t)
|
||||
connector := testConnector(tenantFolderPathFixture())
|
||||
items, err := connector.FetchAll(
|
||||
context.Background(),
|
||||
tenantFolderPathTestConfig(tenantWorkspaceID),
|
||||
[]string{tenantWorkspaceID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
root := tenantWorkspaceName + "/"
|
||||
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
|
||||
"d-easm-faq": root + "EASM-外部攻击面管理模块/EASM 模块使用 FAQ.md",
|
||||
"d-easm-manual": root + "EASM-外部攻击面管理模块/EASM 模块操作手册.docx",
|
||||
"d-cloud-faq": root + "天穹云端安全能力中枢常见问题 FAQ.md",
|
||||
"d-tianxi-faq": root + "天析应急分析/天析应急分析 FAQ.md",
|
||||
"d-tianxi-deck": root + "天析应急分析/天析应急分析介绍.pptx",
|
||||
"d-tianyu-manual": root + "天禹漏洞情报/天禹漏洞情报使用手册.docx",
|
||||
"d-tianyu-deck": root + "天禹漏洞情报/天禹漏洞情报介绍.pptx",
|
||||
"d-tianzhu-manual": root + "天诛渗透攻防/天诛渗透攻防手册.docx",
|
||||
"d-tianzhu-faq-1": root + "天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
|
||||
"d-tianzhu-faq-2": root + "天诛渗透攻防/天诛渗透攻防 FAQ 二.md",
|
||||
"d-tianzhu-2024-faq": root + "天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md",
|
||||
})
|
||||
}
|
||||
|
||||
// The switch has to read the same string spellings the connector's other switch
|
||||
// reads, and anything unrecognised — including an absent key — has to leave the
|
||||
// historical behaviour alone.
|
||||
func TestDingTalkFolderPathsReadsTheSameSettingsSpellings(t *testing.T) {
|
||||
captureLogs(t)
|
||||
for _, testCase := range []struct {
|
||||
label string
|
||||
settings map[string]interface{}
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"key absent",
|
||||
map[string]interface{}{"include_uploaded_files": true},
|
||||
"天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"bool true",
|
||||
map[string]interface{}{"preserve_folder_structure": true},
|
||||
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"string yes",
|
||||
map[string]interface{}{"preserve_folder_structure": "yes"},
|
||||
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"string ON with padding",
|
||||
map[string]interface{}{"preserve_folder_structure": " ON "},
|
||||
tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"bool false",
|
||||
map[string]interface{}{"preserve_folder_structure": false},
|
||||
"天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"unknown spelling",
|
||||
map[string]interface{}{"preserve_folder_structure": "sometimes"},
|
||||
"天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
{
|
||||
"unexpected type",
|
||||
map[string]interface{}{"preserve_folder_structure": 42},
|
||||
"天诛渗透攻防 FAQ 一.md",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
cfg := tenantUploadedFilesConfig(tenantWorkspaceID)
|
||||
cfg.Settings = testCase.settings
|
||||
connector := testConnector(tenantFolderPathFixture())
|
||||
items, err := connector.FetchAll(
|
||||
context.Background(), cfg, []string{tenantWorkspaceID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
names := fileNamesByDocument(t, items)
|
||||
if names["d-tianzhu-faq-1"] != testCase.want {
|
||||
t.Fatalf("FileName = %q, want %q", names["d-tianzhu-faq-1"], testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Selecting a folder is the common case, and the path a document lands on must
|
||||
// not depend on how deep the operator drilled into the picker: the same
|
||||
// document selected through the whole workspace, through its own folder and as
|
||||
// a single document has to end up in one folder, not three.
|
||||
func TestDingTalkFolderPathsDoNotDependOnTheSelectedScope(t *testing.T) {
|
||||
captureLogs(t)
|
||||
connector := testConnector(tenantFolderPathFixture())
|
||||
wholeWorkspace, err := connector.FetchAll(
|
||||
context.Background(),
|
||||
tenantFolderPathTestConfig(tenantWorkspaceID),
|
||||
[]string{tenantWorkspaceID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("workspace scope error = %v", err)
|
||||
}
|
||||
folderID, err := encodeResourceReference(resourceReference{
|
||||
WorkspaceID: tenantWorkspaceID, NodeID: "f-tianzhu-2024", Ancestors: []string{"f-tianzhu"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nestedFolder, err := connector.FetchAll(
|
||||
context.Background(),
|
||||
tenantFolderPathTestConfig(folderID),
|
||||
[]string{folderID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("folder scope error = %v", err)
|
||||
}
|
||||
documentID, err := encodeResourceReference(resourceReference{
|
||||
WorkspaceID: tenantWorkspaceID, NodeID: "d-tianzhu-2024-faq",
|
||||
Ancestors: []string{"f-tianzhu", "f-tianzhu-2024"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
singleDocument, err := connector.FetchAll(
|
||||
context.Background(),
|
||||
tenantFolderPathTestConfig(documentID),
|
||||
[]string{documentID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("document scope error = %v", err)
|
||||
}
|
||||
|
||||
want := tenantWorkspaceName + "/天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md"
|
||||
for label, items := range map[string][]types.FetchedItem{
|
||||
"workspace": wholeWorkspace,
|
||||
"folder": nestedFolder,
|
||||
"document": singleDocument,
|
||||
} {
|
||||
if got := fileNamesByDocument(t, items)["d-tianzhu-2024-faq"]; got != want {
|
||||
t.Errorf("%s scope FileName = %q, want %q", label, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Names are hostile but not tidy: a folder name may contain the path separator,
|
||||
// punctuation DingTalk allows and file systems do not, and two sibling folders
|
||||
// may share a name. Each level is sanitised on its own so a "/" inside one name
|
||||
// cannot invent an extra level, and same-named siblings merge into one folder —
|
||||
// the same trade-off the Yuque connector documents for same-named groups.
|
||||
func TestDingTalkFolderPathsSanitizeEachLevelAndMergeSameNamedFolders(t *testing.T) {
|
||||
captureLogs(t)
|
||||
const workspaceName = "安全/中心"
|
||||
adoc := func(id, name string) node {
|
||||
return node{
|
||||
ID: id, WorkspaceID: "ws-odd", Type: "FILE", Category: "ALIDOC",
|
||||
Extension: "adoc", Name: name,
|
||||
}
|
||||
}
|
||||
folder := func(id, name string) node {
|
||||
return node{
|
||||
ID: id, WorkspaceID: "ws-odd", Type: "FOLDER", Name: name, HasChildren: true,
|
||||
}
|
||||
}
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "ws-odd", RootNodeID: "root-odd", Name: workspaceName}},
|
||||
nodes: map[string][]node{
|
||||
"root-odd": {
|
||||
folder("f-slash", "A/B"),
|
||||
folder("f-punct", `<>:"|?*`),
|
||||
// Two folders with the same title: DingTalk node IDs are
|
||||
// unique, titles are not.
|
||||
folder("f-dup-1", "重复"),
|
||||
folder("f-dup-2", "重复"),
|
||||
// A name of only whitespace falls back to the node ID, which
|
||||
// is what the resource picker shows for it too.
|
||||
folder("f-unnamed", " "),
|
||||
},
|
||||
"f-slash": {adoc("d-slash", "A/B 手册")},
|
||||
"f-punct": {adoc("d-punct", "符号")},
|
||||
"f-dup-1": {adoc("d-dup-1", "第一份")},
|
||||
"f-dup-2": {adoc("d-dup-2", "第二份")},
|
||||
"f-unnamed": {adoc("d-unnamed", "无名目录")},
|
||||
},
|
||||
blocks: map[string][]json.RawMessage{
|
||||
"d-slash": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"a"}}`)},
|
||||
"d-punct": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"b"}}`)},
|
||||
"d-dup-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"c"}}`)},
|
||||
"d-dup-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"d"}}`)},
|
||||
"d-unnamed": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"e"}}`)},
|
||||
},
|
||||
nodeErrors: make(map[string]error),
|
||||
blockErrors: make(map[string]error),
|
||||
}
|
||||
connector := testConnector(api)
|
||||
items, err := connector.FetchAll(
|
||||
context.Background(), tenantFolderPathTestConfig("ws-odd"), []string{"ws-odd"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
// "安全/中心" cannot invent a level either, and neither can "A/B" or the
|
||||
// document name: the file name keeps the sanitised form it already had.
|
||||
assertFileNames(t, fileNamesByDocument(t, items), map[string]string{
|
||||
"d-slash": "安全_中心/A_B/A_B 手册.md",
|
||||
"d-punct": "安全_中心/_______/符号.md",
|
||||
"d-dup-1": "安全_中心/重复/第一份.md",
|
||||
"d-dup-2": "安全_中心/重复/第二份.md",
|
||||
"d-unnamed": "安全_中心/f-unnamed/无名目录.md",
|
||||
})
|
||||
}
|
||||
|
||||
// Turning the switch on in an existing data source is an ingest-path change,
|
||||
// not a migration: an incremental sync compares content revisions, so the
|
||||
// documents already in the knowledge base are not re-read and therefore keep
|
||||
// the flat file names they were ingested with — and, above all, the folder
|
||||
// dimension must never make the connector report them as deleted.
|
||||
func TestDingTalkEnablingFolderPathsDoesNotRelocateOrDeleteExistingDocuments(t *testing.T) {
|
||||
captureLogs(t)
|
||||
connector := testConnector(tenantFolderPathFixture())
|
||||
items, cursor, err := connector.FetchIncremental(
|
||||
context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("first FetchIncremental() error = %v", err)
|
||||
}
|
||||
if names := fileNamesByDocument(t, items); names["d-tianzhu-faq-1"] != "天诛渗透攻防 FAQ 一.md" {
|
||||
t.Fatalf("first sync FileName = %q, want the flat name", names["d-tianzhu-faq-1"])
|
||||
}
|
||||
|
||||
items, _, err = connector.FetchIncremental(
|
||||
context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), cursor,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("second FetchIncremental() error = %v", err)
|
||||
}
|
||||
for _, item := range items {
|
||||
if item.IsDeleted {
|
||||
t.Fatalf("enabling folder paths reported document %q as deleted", item.ExternalID)
|
||||
}
|
||||
}
|
||||
if len(items) != 0 {
|
||||
t.Fatalf("second sync returned %d items, want none: unchanged documents are not "+
|
||||
"re-read, so they keep the file name they were ingested with", len(items))
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,14 @@ type documentSettings struct {
|
|||
// IncludeUploadedFiles admits uploaded Office/PDF files (FILE/DOCUMENT
|
||||
// nodes) that are downloaded from 钉盘 and parsed by WeKnora itself.
|
||||
IncludeUploadedFiles bool
|
||||
// PreserveFolderStructure encodes each document's DingTalk folder hierarchy
|
||||
// into its FileName, so the document lands in the knowledge base folder
|
||||
// that mirrors where it lives in the wiki instead of at the root.
|
||||
//
|
||||
// It is an ingest-path switch, not a migration. An already-ingested
|
||||
// document is only re-read when its content changes, so turning the switch
|
||||
// on does not relocate it; a full sync does.
|
||||
PreserveFolderStructure bool
|
||||
}
|
||||
|
||||
// parseDocumentSettings reads the settings bag of a data source. A missing or
|
||||
|
|
@ -48,6 +56,7 @@ func parseDocumentSettings(dataSourceConfig *types.DataSourceConfig) documentSet
|
|||
raw = dataSourceConfig.Settings
|
||||
}
|
||||
return documentSettings{
|
||||
IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),
|
||||
IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),
|
||||
PreserveFolderStructure: settingBool(raw["preserve_folder_structure"], false),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -233,6 +233,12 @@
|
|||
- **上传文件(默认关闭)**:`Settings.include_uploaded_files`(默认 `false`)开启后,所选范围内钉盘中的普通上传附件(`FILE/DOCUMENT` 的 `docx` / `pptx` / `xlsx` / `pdf`)会被下载并交给 WeKnora 自带解析器入库,文件名保留原扩展名、按扩展名选择解析器;单个文件上限 64 MiB,超出上限即失败且不重试(重签名只会拿到同样的字节)。下载失败(401/403/404 等)会重新申请签名后重试,最多 3 次。
|
||||
- **开关的默认值与成本**:`include_uploaded_files` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后的下一次同步(含增量同步)行为不变**,不会突然开始下载和向量化附件。打开后,这些文件的正文会进入知识库,占用存储并产生 embedding 成本;不需要时请保持关闭。关闭时连接校验、资源列表和同步都会跳过上传附件,同步日志给出的原因是 `include_uploaded_files` 未启用,而不是类型不支持。
|
||||
- **开关的作用点**:测试连接(校验)、资源选择列表、全量同步与增量同步共用同一个判据,一处关闭即处处一致。
|
||||
- **目录结构(默认关闭)**:钉钉知识库本身就是目录树,但连接器历来把文档平铺到知识库根目录,`folder_path` 为空,不同来源目录的文档在知识库里无法区分。`Settings.preserve_folder_structure`(默认 `false`)开启后,`FileName` 编码为 `<工作区名>/<子目录>/…/<标题>.ext`,由下游 `SplitKnowledgeRelativePath` → `NormalizeKnowledgeFolderPath` 拆出 `folder_path`——与语雀 `folder_mode=toc`、飞书云盘目录映射是同一套机制。**工作区名是第一段**:一个知识库可以同时链接多个钉钉数据源,没有这一段,不同工作区里同名的顶层目录会静默合并成一个(语雀用书名作第一段是同样的理由);工作区没有名字时退回用它的 ID。**路径始终以工作区为根**:选整个工作区、选某个子目录、选单篇文档,同一篇文档都落到同一个目录,不会出现两份。每一层目录名单独清洗(名字里的 `/`、`\`、`:` 等替换为 `_`),因此目录名里的斜杠不会凭空多出一层;同名子目录合并为同一个知识库目录(钉钉节点 ID 唯一、标题不唯一),`.`、`..` 这类非法层级由下游归一化丢弃而不是让同步失败。开启后目录由连接器托管:在知识库里手动调整过的目录,会在该文档下次同步时被覆盖回钉钉的结构。
|
||||
- **目录开关的默认值与作用范围**:`preserve_folder_structure` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后 `FileName` 逐字节不变、`folder_path` 仍为空**。它只决定**入库路径**,不决定哪些文档入库:测试连接与资源选择列表不受它影响,关掉它也不会让任何文档消失。目录维度不写入同步游标——开关开与关时游标里的"资源 → 文档版本"表完全一致,所以目录永远不会让文档被判定为"源端已删除"。
|
||||
- **目录相关注意 / 限制**:
|
||||
- **开启开关不会自动归位已导入的文档**:增量同步按 `modifiedTimestamp` 判断变化,只重新读取内容有变化的文档,因此**已经在知识库里的文档会保持原来的扁平文件名**——它们并没有被重新读取。只有内容变更过的文档、或手动触发一次全量同步之后,才会整体移到新目录。这与语雀修改 `folder_mode` 后的行为一致。
|
||||
- **`include_uploaded_files` 关闭时**,上传附件(`docx` / `pptx` / `xlsx` / `pdf`)不入库,其所在目录自然也不会出现;在线 `adoc` 文档不受该开关影响。
|
||||
- **目录深度与长度由下游统一裁剪**:`folder_path` 最多 16 层、单个目录名最多 128 字节、整条路径最多 1024 字节,超出部分被截断(与文件夹上传同一套规则)。
|
||||
- **同步**:按文档 `modifiedTimestamp`(毫秒)增量读取,缺失时回退 `modifiedTime`;合并重叠选择。全量同步也会对照上次游标对账删除,避免 `sync_mode=full` 漏删。目录遍历不完整时暂缓删除。
|
||||
- **正文**:公开 Blocks API 只返回文档根下的一级块;高亮块等容器若响应里带有 `children` 会继续渲染,否则在元数据中标记 `nested_blocks_unavailable`,避免把残缺正文当成完整成功。
|
||||
- **校验**:测试连接会列出知识库、探测根节点列表,并在根下存在在线文档时试读 Blocks;对已开启摄取的上传文件只解析存储位置并申请下载 URL,不下载正文,以便尽早发现缺少 `Wiki.Node.Read` / `Storage.File.Read`,同时让校验对多兆字节文件保持廉价。
|
||||
|
|
|
|||
Loading…
Reference in New Issue