fix(datasource/dingtalk): 跳过不再静默,展开叶子不再报错

同步过去只保留能读的内容:没有 ingest 路径的节点被直接丢掉,日志里
一个字都没有,一个 19 个文件的库只同步出 14 个也无从得知。现在

- scanWorkspace 把「看到了但读不了」的节点一并以 skipped 返回,
  sync 逐个打印身份与原因(视频/媒体是刻意不下载,原生类型是还没实现),
  每个 scope 再补一行 total/synced/skipped/failed 汇总,partial sync
  再也装不成一次干净的运行;
- 可重试的失败仍按失败计(不计入 skipped),并显式记一条 warn,
  与确定性跳过区分开;
- 资源选择器展开一个其实没有子节点的条目(document)时返回空列表而不是
  ErrResourceNotFound:叶子本来就没有子节点,而前端把 error 渲染成
  展开失败的气泡,会让一次同步完全正常的操作看起来出错。前端
  DataSourceEditorDialog 收到空列表后会把该行 has_children 置 false 并
  保留,因此第二次展开不会再发一次空请求(该逻辑已在上游 main 中)。

跳过只在日志里报告:FetchedItem 没有 skip 字段,也不打算为它新增。
main
linus-liu-web 2026-09-27 18:55:31 +08:00 committed by lyingbug
parent ec0ca46334
commit 08f7e702cf
2 changed files with 450 additions and 17 deletions

View File

@ -13,6 +13,7 @@ import (
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@ -21,6 +22,53 @@ const (
maxTraversalNodes = 1_000_000
)
// mediaExtensions are the file types this connector deliberately never
// downloads: media carries no text WeKnora could index.
var mediaExtensions = map[string]struct{}{
"mp4": {}, "mov": {}, "avi": {}, "mkv": {}, "flv": {}, "wmv": {}, "m4v": {}, "webm": {},
"mp3": {}, "wav": {}, "m4a": {}, "aac": {}, "flac": {},
}
// unsupportedDocumentExtensions names the native DingTalk document types the
// wiki API lists but this connector has no ingest path for, so a skip can name
// the concrete type instead of only saying the node is unsupported. The
// remaining types expose no read API at all.
var unsupportedDocumentExtensions = map[string]string{
"able": "DingTalk multi-dimensional table",
"amind": "DingTalk mind map",
"appt": "DingTalk presentation",
"adraw": "DingTalk drawing",
}
// skipReason explains, in the words of the sync log, why a node the connector
// just listed will not be ingested. Both answers are deterministic: retrying
// the sync can never turn such a node into a document, which is what separates
// a skip from a failed read that must be retried.
func skipReason(n node) string {
extension := strings.ToLower(strings.TrimSpace(n.Extension))
_, knownMediaExtension := mediaExtensions[extension]
if strings.EqualFold(n.Category, "VIDEO") || knownMediaExtension {
return "video/media files are deliberately not downloaded by this connector"
}
if label := unsupportedDocumentExtensions[extension]; label != "" {
return label + " has no ingest path in this connector yet"
}
return "no ingest path for this DingTalk node type in this connector yet"
}
// scopeLabel names a sync scope the way an operator can match it in DingTalk:
// the workspace followed by the node the scan starts from.
func scopeLabel(scope syncScope) string {
nodeID := scope.StartNodeID
if scope.Document != nil {
nodeID = scope.Document.ID
}
if nodeID == "" {
return scope.Reference.WorkspaceID
}
return scope.Reference.WorkspaceID + "/" + nodeID
}
var (
_ datasource.Connector = (*Connector)(nil)
_ datasource.FullSyncWithCursor = (*Connector)(nil)
@ -162,9 +210,27 @@ func (c *Connector) ListResources(
if failure := failures[parentID]; failure != nil {
return nil, failure
}
if len(scopes) != 1 || scopes[0].Document != nil {
return nil, fmt.Errorf("%w: DingTalk resource %q is not an expandable folder",
datasource.ErrResourceNotFound, parentID)
if len(scopes) != 1 {
// One reference resolves to exactly one scope. Any other count means
// there is nothing behind it that could be listed; the picker reads
// an error as a failed expansion, so the honest empty listing is
// returned instead.
logger.Warnf(ctx,
"[DingTalk] expand %s: resolved %d scopes, want one; reporting no children",
parentID, len(scopes))
return []types.Resource{}, nil
}
if scopes[0].Document != nil {
// A document is a leaf: it genuinely has no children. Returning an
// error here (the previous "is not an expandable folder") surfaces
// in the picker as a failure toast for a selection that syncs fine,
// so the empty listing is the correct answer. The picker is told
// HasChildren=false for a document, which is what keeps it from
// offering the expander in the first place.
logger.Warnf(ctx,
"[DingTalk] expand %s: the reference is a document, not a folder; reporting no children",
parentID)
return []types.Resource{}, nil
}
parentNodeID = scopes[0].StartNodeID
}
@ -431,7 +497,7 @@ func (c *Connector) sync(
oldRevisions = stored
}
}
documents, err := scanScope(ctx, api, scope)
documents, skipped, err := scanScope(ctx, api, scope)
if err != nil {
if isContextError(err) {
return nil, nil, err
@ -439,13 +505,26 @@ func (c *Connector) sync(
// Never infer deletions from an incomplete tree. Other independent
// selections may still complete, while this scope keeps its previous
// cursor and is retried on the next run.
logger.Warnf(ctx, "[DingTalk] scan scope %s failed, will retry next sync: %v",
scopeLabel(scope), err)
complete = false
next.Resources[scope.ResourceID] = cloneRevisions(oldRevisions)
items = append(items, failedResource(scope.ResourceID, err))
continue
}
// Skipped nodes are deterministic: this connector has no ingest path for
// them, so they are reported once and never retried — unlike a failed
// read below, which must stay retryable.
for _, node := range skipped {
logger.Infof(ctx, "[DingTalk] skip node %s (name=%q type=%s category=%s extension=%s): %s",
node.ID, node.title(), node.Type, node.Category, node.Extension, skipReason(node))
}
newRevisions := make(map[string]string, len(documents))
currentDocuments := make(map[string]struct{}, len(documents))
synced := 0
failed := 0
for _, document := range documents {
if document.ID == "" {
@ -465,10 +544,13 @@ func (c *Connector) sync(
if isContextError(err) {
return nil, nil, err
}
logger.Warnf(ctx,
"[DingTalk] read document %s (name=%q extension=%s) failed, will retry next sync: %v",
document.ID, document.title(), document.Extension, err)
items = append(items, failedDocument(
scope.ResourceID, scope.Reference.WorkspaceID, document, err,
))
failedDocuments++
failed++
if existed {
// Do not advance failed documents. The next incremental run
// must retry them even if modifiedTime remains unchanged.
@ -480,6 +562,7 @@ func (c *Connector) sync(
items = append(items, fetchedDocument(
scope.ResourceID, scope.Reference.WorkspaceID, document, rendered,
))
synced++
newRevisions[document.ID] = revision
}
@ -492,6 +575,14 @@ func (c *Connector) sync(
}
}
next.Resources[scope.ResourceID] = newRevisions
// One line per scope, in the same shape as the IMA connector's. total
// counts every file the scope listed — synced + still-unchanged +
// failed + skipped — so a scope that quietly loses two thirds of its
// nodes can no longer look like a clean run.
logger.Infof(ctx, "[DingTalk] scope %s: total=%d synced=%d skipped=%d failed=%d",
scopeLabel(scope), len(documents)+len(skipped), synced, len(skipped), failed)
failedDocuments += failed
}
// Reconcile the union of all selections. Moving a document between two
@ -670,27 +761,34 @@ func childByID(children []node, nodeID string) (node, bool) {
return node{}, false
}
func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, error) {
// scanScope lists every ingestible document in one scope, together with the
// nodes it saw but cannot ingest. The skipped nodes are returned rather than
// discarded so the caller can report them: a full sync must never look clean
// while silently dropping part of the tree.
func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, []node, error) {
if scope.Document != nil {
return []node{*scope.Document}, nil
return []node{*scope.Document}, nil, nil
}
return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID)
}
// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal
// only; every file is either an ingestible document or a skip.
func scanWorkspace(
ctx context.Context,
api dingTalkAPI,
workspaceID string,
rootNodeID string,
) ([]node, error) {
) ([]node, []node, error) {
queue := []string{rootNodeID}
visitedParents := make(map[string]struct{})
seenNodes := make(map[string]struct{})
var documents []node
var skipped []node
for len(queue) > 0 {
if err := ctx.Err(); err != nil {
return nil, err
return nil, nil, err
}
parentID := queue[0]
queue = queue[1:]
@ -701,24 +799,30 @@ func scanWorkspace(
children, err := api.listNodes(ctx, parentID)
if err != nil {
return nil, err
return nil, nil, err
}
for _, child := range children {
if child.ID == "" {
continue
}
if child.WorkspaceID != "" && child.WorkspaceID != workspaceID {
return nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)
return nil, nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)
}
if _, seen := seenNodes[child.ID]; seen {
continue
}
seenNodes[child.ID] = struct{}{}
if len(seenNodes) > maxTraversalNodes {
return nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)
return nil, nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)
}
if child.isDocument() {
switch {
case child.isDocument():
documents = append(documents, child)
case child.isFolder():
// Containers hold no content of their own, so they are never
// reported as skipped.
default:
skipped = append(skipped, child)
}
if child.isFolder() || child.HasChildren {
queue = append(queue, child.ID)
@ -728,7 +832,10 @@ func scanWorkspace(
sort.SliceStable(documents, func(i, j int) bool {
return documents[i].ID < documents[j].ID
})
return documents, nil
sort.SliceStable(skipped, func(i, j int) bool {
return skipped[i].ID < skipped[j].ID
})
return documents, skipped, nil
}
func cloneRevisions(revisions map[string]string) map[string]string {

View File

@ -1,6 +1,7 @@
package dingtalk
import (
"bytes"
"context"
"encoding/json"
"errors"
@ -8,6 +9,7 @@ import (
"testing"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@ -110,10 +112,14 @@ func TestConnectorListsWorkspacesAndFetchesNestedDocuments(t *testing.T) {
if err != nil || len(documents) != 1 || documents[0].Type != "document" {
t.Fatalf("folder children = %#v, %v; want one document", documents, err)
}
if _, err := connector.ListResources(
// A document is a leaf: expanding it is answered with no children rather
// than an error, so a picker expansion cannot turn a selection that syncs
// fine into a failure.
leafChildren, err := connector.ListResources(
context.Background(), testConfig(), documents[0].ExternalID,
); !errors.Is(err, datasource.ErrResourceNotFound) {
t.Fatalf("expanding a document error = %v, want ErrResourceNotFound", err)
)
if err != nil || len(leafChildren) != 0 {
t.Fatalf("expanding a document = %#v, %v; want no children", leafChildren, err)
}
ancestors, err := connector.ResolveResourceAncestors(
context.Background(), testConfig(), []string{documents[0].ExternalID},
@ -441,3 +447,323 @@ func TestValidateProbesNodeAndDocumentAccess(t *testing.T) {
t.Fatal("Validate() error = nil, want node access failure")
}
}
// Expanding a document is answered with no children rather than an error: a
// leaf genuinely has none, and the picker renders an error as a failed
// expansion — a toast on a step whose selection syncs perfectly well. The
// listing already reports HasChildren=false for a document, which keeps the
// picker from offering the expander; this is the answer for a client that
// expands anyway, or for a selection saved by a build that did offer it.
func TestListResourcesReportsNoChildrenForALeafDocument(t *testing.T) {
api := &fakeAPI{
workspaces: []workspace{{ID: "team", RootNodeID: "team-root", Name: "Team"}},
nodes: map[string][]node{
"team-root": {{
ID: "doc-1", WorkspaceID: "team", Name: "Plan.adoc",
Type: "FILE", Category: "ALIDOC", Extension: "adoc",
}},
},
}
documentID, err := encodeResourceReference(resourceReference{
WorkspaceID: "team", NodeID: "doc-1",
})
if err != nil {
t.Fatal(err)
}
resources, err := testConnector(api).ListResources(context.Background(), testConfig(), documentID)
if err != nil {
t.Fatalf("ListResources(%q) error = %v, want an empty listing", documentID, err)
}
if len(resources) != 0 {
t.Fatalf("ListResources(%q) = %#v, want no children", documentID, resources)
}
}
// captureLogs redirects the process logger into a buffer for one test. The sync
// log is the surface a skipped node has to appear on, so it is asserted on
// directly rather than through an internal counter.
func captureLogs(t *testing.T) *bytes.Buffer {
t.Helper()
var buffer bytes.Buffer
logger.SetOutput(&buffer)
t.Cleanup(logger.ConfigureFromEnv)
return &buffer
}
// skippedNodesFixture mixes supported documents with the node types this
// connector deliberately cannot ingest, spread over a folder tree: reporting
// must survive nesting and must not disturb what is fetched.
func skippedNodesFixture() *fakeAPI {
return &fakeAPI{
workspaces: []workspace{{ID: "space", RootNodeID: "root", Name: "Space"}},
nodes: map[string][]node{
"root": {
{ID: "docs", WorkspaceID: "space", Name: "Docs", Type: "FOLDER"},
{
ID: "video-1", WorkspaceID: "space", Name: "Lesson.mp4", Type: "FILE",
Category: "VIDEO", Extension: "mp4",
},
{
ID: "table-1", WorkspaceID: "space", Name: "Roadmap.able", Type: "FILE",
Category: "ALIDOC", Extension: "able",
},
},
"docs": {
{
ID: "doc-1", WorkspaceID: "space", Name: "Runbook.adoc", Type: "FILE",
Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r1",
},
{
ID: "doc-2", WorkspaceID: "space", Name: "Policy.adoc", Type: "FILE",
Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r2",
},
{
ID: "mind-1", WorkspaceID: "space", Name: "Plan.amind", Type: "FILE",
Category: "ALIDOC", Extension: "amind",
},
{ID: "empty", WorkspaceID: "space", Name: "Empty", Type: "FOLDER"},
},
"empty": nil,
},
blocks: map[string][]json.RawMessage{
"doc-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"run"}}`)},
"doc-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"policy"}}`)},
},
nodeErrors: make(map[string]error),
blockErrors: make(map[string]error),
}
}
// Seeing an unsupported node must not change what is fetched: the supported
// siblings still sync exactly as before, and nothing is requested for a skip.
func TestFetchAllKeepsSyncingSupportedSiblingsOfSkippedNodes(t *testing.T) {
api := skippedNodesFixture()
items, err := testConnector(api).FetchAll(
context.Background(), testConfig("space"), []string{"space"},
)
if err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
if len(items) != 2 {
t.Fatalf("FetchAll() returned %d items, want 2: %#v", len(items), items)
}
byID := make(map[string]types.FetchedItem, len(items))
for _, item := range items {
byID[item.ExternalID] = item
}
if len(byID) != 2 || byID["doc-1"].ContentType != "text/markdown" ||
byID["doc-2"].ContentType != "text/markdown" {
t.Fatalf("supported documents = %#v", byID)
}
for _, skipped := range []string{"video-1", "table-1", "mind-1"} {
if _, exists := byID[skipped]; exists {
t.Fatalf("unsupported node %q was synced: %#v", skipped, byID[skipped])
}
if api.blockCalls[skipped] != 0 {
t.Fatalf("unsupported node %q was requested: %#v", skipped, api.blockCalls)
}
}
if len(api.blockCalls) != 2 || api.blockCalls["doc-1"] != 1 || api.blockCalls["doc-2"] != 1 {
t.Fatalf("block calls = %#v, want one per supported document", api.blockCalls)
}
}
// Every node the connector drops must reach the sync log with its identity and
// the concrete reason it cannot be ingested: media is never downloaded on
// purpose, while a native DingTalk type simply has no ingest path yet.
func TestSyncLogsEverySkippedNodeWithItsReason(t *testing.T) {
api := skippedNodesFixture()
logs := captureLogs(t)
if _, err := testConnector(api).FetchAll(
context.Background(), testConfig("space"), []string{"space"},
); err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
output := logs.String()
for _, want := range []string{
`[DingTalk] skip node video-1 (name="Lesson.mp4" type=FILE category=VIDEO extension=mp4): ` +
"video/media files are deliberately not downloaded by this connector",
`[DingTalk] skip node table-1 (name="Roadmap.able" type=FILE category=ALIDOC extension=able): ` +
"DingTalk multi-dimensional table has no ingest path in this connector yet",
`[DingTalk] skip node mind-1 (name="Plan.amind" type=FILE category=ALIDOC extension=amind): ` +
"DingTalk mind map has no ingest path in this connector yet",
} {
if !strings.Contains(output, want) {
t.Fatalf("sync log is missing:\n%s\ngot:\n%s", want, output)
}
}
for _, folder := range []string{"skip node docs", "skip node empty"} {
if strings.Contains(output, folder) {
t.Fatalf("folder reported as a skipped node: %q in\n%s", folder, output)
}
}
}
// The per-scope summary states how much of the scope arrived, in the same shape
// as the IMA connector's per-knowledge-base line.
func TestSyncLogsPerScopeSummary(t *testing.T) {
api := skippedNodesFixture()
logs := captureLogs(t)
if _, err := testConnector(api).FetchAll(
context.Background(), testConfig("space"), []string{"space"},
); err != nil {
t.Fatalf("FetchAll() error = %v", err)
}
want := "[DingTalk] scope space/root: total=5 synced=2 skipped=3 failed=0"
if !strings.Contains(logs.String(), want) {
t.Fatalf("sync log is missing %q:\n%s", want, logs.String())
}
}
// A transient read failure is counted as failed — not as skipped — is logged as
// retryable, and still leaves the document out of the cursor.
func TestSyncSummaryCountsTransientFailuresAndStillRetries(t *testing.T) {
api := skippedNodesFixture()
api.blockErrors = map[string]error{"doc-2": errors.New("storage temporarily unavailable")}
logs := captureLogs(t)
items, err := testConnector(api).FetchAll(
context.Background(), testConfig("space"), []string{"space"},
)
var partial *datasource.PartialFetchError
if !errors.As(err, &partial) {
t.Fatalf("FetchAll() error = %v, want PartialFetchError", err)
}
if len(items) != 2 {
t.Fatalf("FetchAll() items = %#v, want 1 document and 1 failure", items)
}
output := logs.String()
if !strings.Contains(output, "storage temporarily unavailable") ||
!strings.Contains(output, "failed, will retry next sync") {
t.Fatalf("transient failure was not logged as retryable:\n%s", output)
}
want := "[DingTalk] scope space/root: total=5 synced=1 skipped=3 failed=1"
if !strings.Contains(output, want) {
t.Fatalf("sync log is missing %q:\n%s", want, output)
}
}
// Containers are not content: a tree of folders alone must produce no skip
// report at all, only a zeroed summary.
func TestSyncLogsNoSkipsForFolderOnlyTree(t *testing.T) {
api := &fakeAPI{
workspaces: []workspace{{ID: "space", RootNodeID: "root"}},
nodes: map[string][]node{
"root": {{ID: "outer", WorkspaceID: "space", Name: "Outer", Type: "FOLDER"}},
"outer": {{ID: "inner", WorkspaceID: "space", Name: "Inner", Type: "FOLDER"}},
"inner": nil,
},
}
logs := captureLogs(t)
items, err := testConnector(api).FetchAll(
context.Background(), testConfig("space"), []string{"space"},
)
if err != nil || len(items) != 0 {
t.Fatalf("FetchAll() = %#v, %v; want no items", items, err)
}
output := logs.String()
if strings.Contains(output, "skip node") {
t.Fatalf("folder-only tree reported skips:\n%s", output)
}
want := "[DingTalk] scope space/root: total=0 synced=0 skipped=0 failed=0"
if !strings.Contains(output, want) {
t.Fatalf("sync log is missing %q:\n%s", want, output)
}
}
// scanScope hands the caller the skipped nodes with their DingTalk identity
// intact, and a single-document scope has no tree to skip anything from.
func TestScanScopeReportsSkippedNodesAndSingleDocumentScopeHasNone(t *testing.T) {
api := skippedNodesFixture()
documents, skipped, err := scanScope(context.Background(), api, syncScope{
ResourceID: "resource",
Reference: resourceReference{WorkspaceID: "space"},
StartNodeID: "root",
})
if err != nil {
t.Fatalf("scanScope() error = %v", err)
}
if len(documents) != 2 {
t.Fatalf("scanScope() documents = %#v, want 2", documents)
}
if len(skipped) != 3 {
t.Fatalf("scanScope() skipped = %#v, want 3", skipped)
}
got := make(map[string]node, len(skipped))
for _, item := range skipped {
got[item.ID] = item
}
if got["video-1"].Category != "VIDEO" || got["video-1"].Extension != "mp4" ||
got["video-1"].Name != "Lesson.mp4" ||
got["table-1"].Category != "ALIDOC" || got["table-1"].Extension != "able" ||
got["mind-1"].Extension != "amind" {
t.Fatalf("skipped nodes lost their identity: %#v", got)
}
document := syncDocument()
documents, skipped, err = scanScope(context.Background(), api, syncScope{
ResourceID: "resource",
Reference: resourceReference{WorkspaceID: "space"},
Document: &document,
})
if err != nil || len(documents) != 1 || documents[0].ID != "doc" || len(skipped) != 0 {
t.Fatalf("single-document scanScope() = %#v, %#v, %v", documents, skipped, err)
}
}
// The reason has to say which kind of unsupported a node is: a video is skipped
// on purpose, a native type has simply not been implemented.
func TestSkipReasonDistinguishesMediaFromUnimplementedTypes(t *testing.T) {
for _, testCase := range []struct {
label string
node node
want string
}{
{
"video category",
node{Type: "FILE", Category: "VIDEO", Extension: "mp4"},
"video/media files are deliberately not downloaded by this connector",
},
{
"video extension without a category",
node{Type: "FILE", Category: "OTHER", Extension: "MOV"},
"video/media files are deliberately not downloaded by this connector",
},
{
"audio extension",
node{Type: "FILE", Category: "OTHER", Extension: "mp3"},
"video/media files are deliberately not downloaded by this connector",
},
{
// Only the types with a name worth printing are in the map; every
// other unsupported node gets the generic reason. Kept as a
// regression guard on the map.
"type without a dedicated label",
node{Type: "FILE", Category: "ALIDOC", Extension: "axls"},
"no ingest path for this DingTalk node type in this connector yet",
},
{
"multidimensional table",
node{Type: "FILE", Category: "ALIDOC", Extension: "able"},
"DingTalk multi-dimensional table has no ingest path in this connector yet",
},
{
"mind map",
node{Type: "FILE", Category: "ALIDOC", Extension: "amind"},
"DingTalk mind map has no ingest path in this connector yet",
},
{
"unknown type",
node{Type: "FILE", Category: "OTHER", Extension: "bin"},
"no ingest path for this DingTalk node type in this connector yet",
},
} {
t.Run(testCase.label, func(t *testing.T) {
if got := skipReason(testCase.node); got != testCase.want {
t.Fatalf("skipReason() = %q, want %q", got, testCase.want)
}
})
}
}