diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.go index 1a8f3b83..798f5fd5 100644 --- a/internal/datasource/connector/dingtalk/connector.go +++ b/internal/datasource/connector/dingtalk/connector.go @@ -13,6 +13,7 @@ import ( "unicode/utf8" "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -21,6 +22,53 @@ const ( maxTraversalNodes = 1_000_000 ) +// mediaExtensions are the file types this connector deliberately never +// downloads: media carries no text WeKnora could index. +var mediaExtensions = map[string]struct{}{ + "mp4": {}, "mov": {}, "avi": {}, "mkv": {}, "flv": {}, "wmv": {}, "m4v": {}, "webm": {}, + "mp3": {}, "wav": {}, "m4a": {}, "aac": {}, "flac": {}, +} + +// unsupportedDocumentExtensions names the native DingTalk document types the +// wiki API lists but this connector has no ingest path for, so a skip can name +// the concrete type instead of only saying the node is unsupported. The +// remaining types expose no read API at all. +var unsupportedDocumentExtensions = map[string]string{ + "able": "DingTalk multi-dimensional table", + "amind": "DingTalk mind map", + "appt": "DingTalk presentation", + "adraw": "DingTalk drawing", +} + +// skipReason explains, in the words of the sync log, why a node the connector +// just listed will not be ingested. Both answers are deterministic: retrying +// the sync can never turn such a node into a document, which is what separates +// a skip from a failed read that must be retried. +func skipReason(n node) string { + extension := strings.ToLower(strings.TrimSpace(n.Extension)) + _, knownMediaExtension := mediaExtensions[extension] + if strings.EqualFold(n.Category, "VIDEO") || knownMediaExtension { + return "video/media files are deliberately not downloaded by this connector" + } + if label := unsupportedDocumentExtensions[extension]; label != "" { + return label + " has no ingest path in this connector yet" + } + return "no ingest path for this DingTalk node type in this connector yet" +} + +// scopeLabel names a sync scope the way an operator can match it in DingTalk: +// the workspace followed by the node the scan starts from. +func scopeLabel(scope syncScope) string { + nodeID := scope.StartNodeID + if scope.Document != nil { + nodeID = scope.Document.ID + } + if nodeID == "" { + return scope.Reference.WorkspaceID + } + return scope.Reference.WorkspaceID + "/" + nodeID +} + var ( _ datasource.Connector = (*Connector)(nil) _ datasource.FullSyncWithCursor = (*Connector)(nil) @@ -162,9 +210,27 @@ func (c *Connector) ListResources( if failure := failures[parentID]; failure != nil { return nil, failure } - if len(scopes) != 1 || scopes[0].Document != nil { - return nil, fmt.Errorf("%w: DingTalk resource %q is not an expandable folder", - datasource.ErrResourceNotFound, parentID) + if len(scopes) != 1 { + // One reference resolves to exactly one scope. Any other count means + // there is nothing behind it that could be listed; the picker reads + // an error as a failed expansion, so the honest empty listing is + // returned instead. + logger.Warnf(ctx, + "[DingTalk] expand %s: resolved %d scopes, want one; reporting no children", + parentID, len(scopes)) + return []types.Resource{}, nil + } + if scopes[0].Document != nil { + // A document is a leaf: it genuinely has no children. Returning an + // error here (the previous "is not an expandable folder") surfaces + // in the picker as a failure toast for a selection that syncs fine, + // so the empty listing is the correct answer. The picker is told + // HasChildren=false for a document, which is what keeps it from + // offering the expander in the first place. + logger.Warnf(ctx, + "[DingTalk] expand %s: the reference is a document, not a folder; reporting no children", + parentID) + return []types.Resource{}, nil } parentNodeID = scopes[0].StartNodeID } @@ -431,7 +497,7 @@ func (c *Connector) sync( oldRevisions = stored } } - documents, err := scanScope(ctx, api, scope) + documents, skipped, err := scanScope(ctx, api, scope) if err != nil { if isContextError(err) { return nil, nil, err @@ -439,13 +505,26 @@ func (c *Connector) sync( // Never infer deletions from an incomplete tree. Other independent // selections may still complete, while this scope keeps its previous // cursor and is retried on the next run. + logger.Warnf(ctx, "[DingTalk] scan scope %s failed, will retry next sync: %v", + scopeLabel(scope), err) complete = false next.Resources[scope.ResourceID] = cloneRevisions(oldRevisions) items = append(items, failedResource(scope.ResourceID, err)) continue } + + // Skipped nodes are deterministic: this connector has no ingest path for + // them, so they are reported once and never retried — unlike a failed + // read below, which must stay retryable. + for _, node := range skipped { + logger.Infof(ctx, "[DingTalk] skip node %s (name=%q type=%s category=%s extension=%s): %s", + node.ID, node.title(), node.Type, node.Category, node.Extension, skipReason(node)) + } + newRevisions := make(map[string]string, len(documents)) currentDocuments := make(map[string]struct{}, len(documents)) + synced := 0 + failed := 0 for _, document := range documents { if document.ID == "" { @@ -465,10 +544,13 @@ func (c *Connector) sync( if isContextError(err) { return nil, nil, err } + logger.Warnf(ctx, + "[DingTalk] read document %s (name=%q extension=%s) failed, will retry next sync: %v", + document.ID, document.title(), document.Extension, err) items = append(items, failedDocument( scope.ResourceID, scope.Reference.WorkspaceID, document, err, )) - failedDocuments++ + failed++ if existed { // Do not advance failed documents. The next incremental run // must retry them even if modifiedTime remains unchanged. @@ -480,6 +562,7 @@ func (c *Connector) sync( items = append(items, fetchedDocument( scope.ResourceID, scope.Reference.WorkspaceID, document, rendered, )) + synced++ newRevisions[document.ID] = revision } @@ -492,6 +575,14 @@ func (c *Connector) sync( } } next.Resources[scope.ResourceID] = newRevisions + + // One line per scope, in the same shape as the IMA connector's. total + // counts every file the scope listed — synced + still-unchanged + + // failed + skipped — so a scope that quietly loses two thirds of its + // nodes can no longer look like a clean run. + logger.Infof(ctx, "[DingTalk] scope %s: total=%d synced=%d skipped=%d failed=%d", + scopeLabel(scope), len(documents)+len(skipped), synced, len(skipped), failed) + failedDocuments += failed } // Reconcile the union of all selections. Moving a document between two @@ -670,27 +761,34 @@ func childByID(children []node, nodeID string) (node, bool) { return node{}, false } -func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, error) { +// scanScope lists every ingestible document in one scope, together with the +// nodes it saw but cannot ingest. The skipped nodes are returned rather than +// discarded so the caller can report them: a full sync must never look clean +// while silently dropping part of the tree. +func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, []node, error) { if scope.Document != nil { - return []node{*scope.Document}, nil + return []node{*scope.Document}, nil, nil } return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID) } +// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal +// only; every file is either an ingestible document or a skip. func scanWorkspace( ctx context.Context, api dingTalkAPI, workspaceID string, rootNodeID string, -) ([]node, error) { +) ([]node, []node, error) { queue := []string{rootNodeID} visitedParents := make(map[string]struct{}) seenNodes := make(map[string]struct{}) var documents []node + var skipped []node for len(queue) > 0 { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } parentID := queue[0] queue = queue[1:] @@ -701,24 +799,30 @@ func scanWorkspace( children, err := api.listNodes(ctx, parentID) if err != nil { - return nil, err + return nil, nil, err } for _, child := range children { if child.ID == "" { continue } if child.WorkspaceID != "" && child.WorkspaceID != workspaceID { - return nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID) + return nil, nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID) } if _, seen := seenNodes[child.ID]; seen { continue } seenNodes[child.ID] = struct{}{} if len(seenNodes) > maxTraversalNodes { - return nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes) + return nil, nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes) } - if child.isDocument() { + switch { + case child.isDocument(): documents = append(documents, child) + case child.isFolder(): + // Containers hold no content of their own, so they are never + // reported as skipped. + default: + skipped = append(skipped, child) } if child.isFolder() || child.HasChildren { queue = append(queue, child.ID) @@ -728,7 +832,10 @@ func scanWorkspace( sort.SliceStable(documents, func(i, j int) bool { return documents[i].ID < documents[j].ID }) - return documents, nil + sort.SliceStable(skipped, func(i, j int) bool { + return skipped[i].ID < skipped[j].ID + }) + return documents, skipped, nil } func cloneRevisions(revisions map[string]string) map[string]string { diff --git a/internal/datasource/connector/dingtalk/connector_test.go b/internal/datasource/connector/dingtalk/connector_test.go index 915d3719..cfd3e182 100644 --- a/internal/datasource/connector/dingtalk/connector_test.go +++ b/internal/datasource/connector/dingtalk/connector_test.go @@ -1,6 +1,7 @@ package dingtalk import ( + "bytes" "context" "encoding/json" "errors" @@ -8,6 +9,7 @@ import ( "testing" "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -110,10 +112,14 @@ func TestConnectorListsWorkspacesAndFetchesNestedDocuments(t *testing.T) { if err != nil || len(documents) != 1 || documents[0].Type != "document" { t.Fatalf("folder children = %#v, %v; want one document", documents, err) } - if _, err := connector.ListResources( + // A document is a leaf: expanding it is answered with no children rather + // than an error, so a picker expansion cannot turn a selection that syncs + // fine into a failure. + leafChildren, err := connector.ListResources( context.Background(), testConfig(), documents[0].ExternalID, - ); !errors.Is(err, datasource.ErrResourceNotFound) { - t.Fatalf("expanding a document error = %v, want ErrResourceNotFound", err) + ) + if err != nil || len(leafChildren) != 0 { + t.Fatalf("expanding a document = %#v, %v; want no children", leafChildren, err) } ancestors, err := connector.ResolveResourceAncestors( context.Background(), testConfig(), []string{documents[0].ExternalID}, @@ -441,3 +447,323 @@ func TestValidateProbesNodeAndDocumentAccess(t *testing.T) { t.Fatal("Validate() error = nil, want node access failure") } } + +// Expanding a document is answered with no children rather than an error: a +// leaf genuinely has none, and the picker renders an error as a failed +// expansion — a toast on a step whose selection syncs perfectly well. The +// listing already reports HasChildren=false for a document, which keeps the +// picker from offering the expander; this is the answer for a client that +// expands anyway, or for a selection saved by a build that did offer it. +func TestListResourcesReportsNoChildrenForALeafDocument(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "team", RootNodeID: "team-root", Name: "Team"}}, + nodes: map[string][]node{ + "team-root": {{ + ID: "doc-1", WorkspaceID: "team", Name: "Plan.adoc", + Type: "FILE", Category: "ALIDOC", Extension: "adoc", + }}, + }, + } + documentID, err := encodeResourceReference(resourceReference{ + WorkspaceID: "team", NodeID: "doc-1", + }) + if err != nil { + t.Fatal(err) + } + + resources, err := testConnector(api).ListResources(context.Background(), testConfig(), documentID) + if err != nil { + t.Fatalf("ListResources(%q) error = %v, want an empty listing", documentID, err) + } + if len(resources) != 0 { + t.Fatalf("ListResources(%q) = %#v, want no children", documentID, resources) + } +} + +// captureLogs redirects the process logger into a buffer for one test. The sync +// log is the surface a skipped node has to appear on, so it is asserted on +// directly rather than through an internal counter. +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buffer bytes.Buffer + logger.SetOutput(&buffer) + t.Cleanup(logger.ConfigureFromEnv) + return &buffer +} + +// skippedNodesFixture mixes supported documents with the node types this +// connector deliberately cannot ingest, spread over a folder tree: reporting +// must survive nesting and must not disturb what is fetched. +func skippedNodesFixture() *fakeAPI { + return &fakeAPI{ + workspaces: []workspace{{ID: "space", RootNodeID: "root", Name: "Space"}}, + nodes: map[string][]node{ + "root": { + {ID: "docs", WorkspaceID: "space", Name: "Docs", Type: "FOLDER"}, + { + ID: "video-1", WorkspaceID: "space", Name: "Lesson.mp4", Type: "FILE", + Category: "VIDEO", Extension: "mp4", + }, + { + ID: "table-1", WorkspaceID: "space", Name: "Roadmap.able", Type: "FILE", + Category: "ALIDOC", Extension: "able", + }, + }, + "docs": { + { + ID: "doc-1", WorkspaceID: "space", Name: "Runbook.adoc", Type: "FILE", + Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r1", + }, + { + ID: "doc-2", WorkspaceID: "space", Name: "Policy.adoc", Type: "FILE", + Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r2", + }, + { + ID: "mind-1", WorkspaceID: "space", Name: "Plan.amind", Type: "FILE", + Category: "ALIDOC", Extension: "amind", + }, + {ID: "empty", WorkspaceID: "space", Name: "Empty", Type: "FOLDER"}, + }, + "empty": nil, + }, + blocks: map[string][]json.RawMessage{ + "doc-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"run"}}`)}, + "doc-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"policy"}}`)}, + }, + nodeErrors: make(map[string]error), + blockErrors: make(map[string]error), + } +} + +// Seeing an unsupported node must not change what is fetched: the supported +// siblings still sync exactly as before, and nothing is requested for a skip. +func TestFetchAllKeepsSyncingSupportedSiblingsOfSkippedNodes(t *testing.T) { + api := skippedNodesFixture() + items, err := testConnector(api).FetchAll( + context.Background(), testConfig("space"), []string{"space"}, + ) + if err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + if len(items) != 2 { + t.Fatalf("FetchAll() returned %d items, want 2: %#v", len(items), items) + } + byID := make(map[string]types.FetchedItem, len(items)) + for _, item := range items { + byID[item.ExternalID] = item + } + if len(byID) != 2 || byID["doc-1"].ContentType != "text/markdown" || + byID["doc-2"].ContentType != "text/markdown" { + t.Fatalf("supported documents = %#v", byID) + } + for _, skipped := range []string{"video-1", "table-1", "mind-1"} { + if _, exists := byID[skipped]; exists { + t.Fatalf("unsupported node %q was synced: %#v", skipped, byID[skipped]) + } + if api.blockCalls[skipped] != 0 { + t.Fatalf("unsupported node %q was requested: %#v", skipped, api.blockCalls) + } + } + if len(api.blockCalls) != 2 || api.blockCalls["doc-1"] != 1 || api.blockCalls["doc-2"] != 1 { + t.Fatalf("block calls = %#v, want one per supported document", api.blockCalls) + } +} + +// Every node the connector drops must reach the sync log with its identity and +// the concrete reason it cannot be ingested: media is never downloaded on +// purpose, while a native DingTalk type simply has no ingest path yet. +func TestSyncLogsEverySkippedNodeWithItsReason(t *testing.T) { + api := skippedNodesFixture() + logs := captureLogs(t) + if _, err := testConnector(api).FetchAll( + context.Background(), testConfig("space"), []string{"space"}, + ); err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + + output := logs.String() + for _, want := range []string{ + `[DingTalk] skip node video-1 (name="Lesson.mp4" type=FILE category=VIDEO extension=mp4): ` + + "video/media files are deliberately not downloaded by this connector", + `[DingTalk] skip node table-1 (name="Roadmap.able" type=FILE category=ALIDOC extension=able): ` + + "DingTalk multi-dimensional table has no ingest path in this connector yet", + `[DingTalk] skip node mind-1 (name="Plan.amind" type=FILE category=ALIDOC extension=amind): ` + + "DingTalk mind map has no ingest path in this connector yet", + } { + if !strings.Contains(output, want) { + t.Fatalf("sync log is missing:\n%s\ngot:\n%s", want, output) + } + } + for _, folder := range []string{"skip node docs", "skip node empty"} { + if strings.Contains(output, folder) { + t.Fatalf("folder reported as a skipped node: %q in\n%s", folder, output) + } + } +} + +// The per-scope summary states how much of the scope arrived, in the same shape +// as the IMA connector's per-knowledge-base line. +func TestSyncLogsPerScopeSummary(t *testing.T) { + api := skippedNodesFixture() + logs := captureLogs(t) + if _, err := testConnector(api).FetchAll( + context.Background(), testConfig("space"), []string{"space"}, + ); err != nil { + t.Fatalf("FetchAll() error = %v", err) + } + want := "[DingTalk] scope space/root: total=5 synced=2 skipped=3 failed=0" + if !strings.Contains(logs.String(), want) { + t.Fatalf("sync log is missing %q:\n%s", want, logs.String()) + } +} + +// A transient read failure is counted as failed — not as skipped — is logged as +// retryable, and still leaves the document out of the cursor. +func TestSyncSummaryCountsTransientFailuresAndStillRetries(t *testing.T) { + api := skippedNodesFixture() + api.blockErrors = map[string]error{"doc-2": errors.New("storage temporarily unavailable")} + logs := captureLogs(t) + + items, err := testConnector(api).FetchAll( + context.Background(), testConfig("space"), []string{"space"}, + ) + var partial *datasource.PartialFetchError + if !errors.As(err, &partial) { + t.Fatalf("FetchAll() error = %v, want PartialFetchError", err) + } + if len(items) != 2 { + t.Fatalf("FetchAll() items = %#v, want 1 document and 1 failure", items) + } + output := logs.String() + if !strings.Contains(output, "storage temporarily unavailable") || + !strings.Contains(output, "failed, will retry next sync") { + t.Fatalf("transient failure was not logged as retryable:\n%s", output) + } + want := "[DingTalk] scope space/root: total=5 synced=1 skipped=3 failed=1" + if !strings.Contains(output, want) { + t.Fatalf("sync log is missing %q:\n%s", want, output) + } +} + +// Containers are not content: a tree of folders alone must produce no skip +// report at all, only a zeroed summary. +func TestSyncLogsNoSkipsForFolderOnlyTree(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "space", RootNodeID: "root"}}, + nodes: map[string][]node{ + "root": {{ID: "outer", WorkspaceID: "space", Name: "Outer", Type: "FOLDER"}}, + "outer": {{ID: "inner", WorkspaceID: "space", Name: "Inner", Type: "FOLDER"}}, + "inner": nil, + }, + } + logs := captureLogs(t) + items, err := testConnector(api).FetchAll( + context.Background(), testConfig("space"), []string{"space"}, + ) + if err != nil || len(items) != 0 { + t.Fatalf("FetchAll() = %#v, %v; want no items", items, err) + } + output := logs.String() + if strings.Contains(output, "skip node") { + t.Fatalf("folder-only tree reported skips:\n%s", output) + } + want := "[DingTalk] scope space/root: total=0 synced=0 skipped=0 failed=0" + if !strings.Contains(output, want) { + t.Fatalf("sync log is missing %q:\n%s", want, output) + } +} + +// scanScope hands the caller the skipped nodes with their DingTalk identity +// intact, and a single-document scope has no tree to skip anything from. +func TestScanScopeReportsSkippedNodesAndSingleDocumentScopeHasNone(t *testing.T) { + api := skippedNodesFixture() + documents, skipped, err := scanScope(context.Background(), api, syncScope{ + ResourceID: "resource", + Reference: resourceReference{WorkspaceID: "space"}, + StartNodeID: "root", + }) + if err != nil { + t.Fatalf("scanScope() error = %v", err) + } + if len(documents) != 2 { + t.Fatalf("scanScope() documents = %#v, want 2", documents) + } + if len(skipped) != 3 { + t.Fatalf("scanScope() skipped = %#v, want 3", skipped) + } + got := make(map[string]node, len(skipped)) + for _, item := range skipped { + got[item.ID] = item + } + if got["video-1"].Category != "VIDEO" || got["video-1"].Extension != "mp4" || + got["video-1"].Name != "Lesson.mp4" || + got["table-1"].Category != "ALIDOC" || got["table-1"].Extension != "able" || + got["mind-1"].Extension != "amind" { + t.Fatalf("skipped nodes lost their identity: %#v", got) + } + + document := syncDocument() + documents, skipped, err = scanScope(context.Background(), api, syncScope{ + ResourceID: "resource", + Reference: resourceReference{WorkspaceID: "space"}, + Document: &document, + }) + if err != nil || len(documents) != 1 || documents[0].ID != "doc" || len(skipped) != 0 { + t.Fatalf("single-document scanScope() = %#v, %#v, %v", documents, skipped, err) + } +} + +// The reason has to say which kind of unsupported a node is: a video is skipped +// on purpose, a native type has simply not been implemented. +func TestSkipReasonDistinguishesMediaFromUnimplementedTypes(t *testing.T) { + for _, testCase := range []struct { + label string + node node + want string + }{ + { + "video category", + node{Type: "FILE", Category: "VIDEO", Extension: "mp4"}, + "video/media files are deliberately not downloaded by this connector", + }, + { + "video extension without a category", + node{Type: "FILE", Category: "OTHER", Extension: "MOV"}, + "video/media files are deliberately not downloaded by this connector", + }, + { + "audio extension", + node{Type: "FILE", Category: "OTHER", Extension: "mp3"}, + "video/media files are deliberately not downloaded by this connector", + }, + { + // Only the types with a name worth printing are in the map; every + // other unsupported node gets the generic reason. Kept as a + // regression guard on the map. + "type without a dedicated label", + node{Type: "FILE", Category: "ALIDOC", Extension: "axls"}, + "no ingest path for this DingTalk node type in this connector yet", + }, + { + "multidimensional table", + node{Type: "FILE", Category: "ALIDOC", Extension: "able"}, + "DingTalk multi-dimensional table has no ingest path in this connector yet", + }, + { + "mind map", + node{Type: "FILE", Category: "ALIDOC", Extension: "amind"}, + "DingTalk mind map has no ingest path in this connector yet", + }, + { + "unknown type", + node{Type: "FILE", Category: "OTHER", Extension: "bin"}, + "no ingest path for this DingTalk node type in this connector yet", + }, + } { + t.Run(testCase.label, func(t *testing.T) { + if got := skipReason(testCase.node); got != testCase.want { + t.Fatalf("skipReason() = %q, want %q", got, testCase.want) + } + }) + } +}