fix(datasource/dingtalk): 跳过不再静默,展开叶子不再报错
同步过去只保留能读的内容:没有 ingest 路径的节点被直接丢掉,日志里 一个字都没有,一个 19 个文件的库只同步出 14 个也无从得知。现在 - scanWorkspace 把「看到了但读不了」的节点一并以 skipped 返回, sync 逐个打印身份与原因(视频/媒体是刻意不下载,原生类型是还没实现), 每个 scope 再补一行 total/synced/skipped/failed 汇总,partial sync 再也装不成一次干净的运行; - 可重试的失败仍按失败计(不计入 skipped),并显式记一条 warn, 与确定性跳过区分开; - 资源选择器展开一个其实没有子节点的条目(document)时返回空列表而不是 ErrResourceNotFound:叶子本来就没有子节点,而前端把 error 渲染成 展开失败的气泡,会让一次同步完全正常的操作看起来出错。前端 DataSourceEditorDialog 收到空列表后会把该行 has_children 置 false 并 保留,因此第二次展开不会再发一次空请求(该逻辑已在上游 main 中)。 跳过只在日志里报告:FetchedItem 没有 skip 字段,也不打算为它新增。main
parent
ec0ca46334
commit
08f7e702cf
|
|
@ -13,6 +13,7 @@ import (
|
|||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
|
|
@ -21,6 +22,53 @@ const (
|
|||
maxTraversalNodes = 1_000_000
|
||||
)
|
||||
|
||||
// mediaExtensions are the file types this connector deliberately never
|
||||
// downloads: media carries no text WeKnora could index.
|
||||
var mediaExtensions = map[string]struct{}{
|
||||
"mp4": {}, "mov": {}, "avi": {}, "mkv": {}, "flv": {}, "wmv": {}, "m4v": {}, "webm": {},
|
||||
"mp3": {}, "wav": {}, "m4a": {}, "aac": {}, "flac": {},
|
||||
}
|
||||
|
||||
// unsupportedDocumentExtensions names the native DingTalk document types the
|
||||
// wiki API lists but this connector has no ingest path for, so a skip can name
|
||||
// the concrete type instead of only saying the node is unsupported. The
|
||||
// remaining types expose no read API at all.
|
||||
var unsupportedDocumentExtensions = map[string]string{
|
||||
"able": "DingTalk multi-dimensional table",
|
||||
"amind": "DingTalk mind map",
|
||||
"appt": "DingTalk presentation",
|
||||
"adraw": "DingTalk drawing",
|
||||
}
|
||||
|
||||
// skipReason explains, in the words of the sync log, why a node the connector
|
||||
// just listed will not be ingested. Both answers are deterministic: retrying
|
||||
// the sync can never turn such a node into a document, which is what separates
|
||||
// a skip from a failed read that must be retried.
|
||||
func skipReason(n node) string {
|
||||
extension := strings.ToLower(strings.TrimSpace(n.Extension))
|
||||
_, knownMediaExtension := mediaExtensions[extension]
|
||||
if strings.EqualFold(n.Category, "VIDEO") || knownMediaExtension {
|
||||
return "video/media files are deliberately not downloaded by this connector"
|
||||
}
|
||||
if label := unsupportedDocumentExtensions[extension]; label != "" {
|
||||
return label + " has no ingest path in this connector yet"
|
||||
}
|
||||
return "no ingest path for this DingTalk node type in this connector yet"
|
||||
}
|
||||
|
||||
// scopeLabel names a sync scope the way an operator can match it in DingTalk:
|
||||
// the workspace followed by the node the scan starts from.
|
||||
func scopeLabel(scope syncScope) string {
|
||||
nodeID := scope.StartNodeID
|
||||
if scope.Document != nil {
|
||||
nodeID = scope.Document.ID
|
||||
}
|
||||
if nodeID == "" {
|
||||
return scope.Reference.WorkspaceID
|
||||
}
|
||||
return scope.Reference.WorkspaceID + "/" + nodeID
|
||||
}
|
||||
|
||||
var (
|
||||
_ datasource.Connector = (*Connector)(nil)
|
||||
_ datasource.FullSyncWithCursor = (*Connector)(nil)
|
||||
|
|
@ -162,9 +210,27 @@ func (c *Connector) ListResources(
|
|||
if failure := failures[parentID]; failure != nil {
|
||||
return nil, failure
|
||||
}
|
||||
if len(scopes) != 1 || scopes[0].Document != nil {
|
||||
return nil, fmt.Errorf("%w: DingTalk resource %q is not an expandable folder",
|
||||
datasource.ErrResourceNotFound, parentID)
|
||||
if len(scopes) != 1 {
|
||||
// One reference resolves to exactly one scope. Any other count means
|
||||
// there is nothing behind it that could be listed; the picker reads
|
||||
// an error as a failed expansion, so the honest empty listing is
|
||||
// returned instead.
|
||||
logger.Warnf(ctx,
|
||||
"[DingTalk] expand %s: resolved %d scopes, want one; reporting no children",
|
||||
parentID, len(scopes))
|
||||
return []types.Resource{}, nil
|
||||
}
|
||||
if scopes[0].Document != nil {
|
||||
// A document is a leaf: it genuinely has no children. Returning an
|
||||
// error here (the previous "is not an expandable folder") surfaces
|
||||
// in the picker as a failure toast for a selection that syncs fine,
|
||||
// so the empty listing is the correct answer. The picker is told
|
||||
// HasChildren=false for a document, which is what keeps it from
|
||||
// offering the expander in the first place.
|
||||
logger.Warnf(ctx,
|
||||
"[DingTalk] expand %s: the reference is a document, not a folder; reporting no children",
|
||||
parentID)
|
||||
return []types.Resource{}, nil
|
||||
}
|
||||
parentNodeID = scopes[0].StartNodeID
|
||||
}
|
||||
|
|
@ -431,7 +497,7 @@ func (c *Connector) sync(
|
|||
oldRevisions = stored
|
||||
}
|
||||
}
|
||||
documents, err := scanScope(ctx, api, scope)
|
||||
documents, skipped, err := scanScope(ctx, api, scope)
|
||||
if err != nil {
|
||||
if isContextError(err) {
|
||||
return nil, nil, err
|
||||
|
|
@ -439,13 +505,26 @@ func (c *Connector) sync(
|
|||
// Never infer deletions from an incomplete tree. Other independent
|
||||
// selections may still complete, while this scope keeps its previous
|
||||
// cursor and is retried on the next run.
|
||||
logger.Warnf(ctx, "[DingTalk] scan scope %s failed, will retry next sync: %v",
|
||||
scopeLabel(scope), err)
|
||||
complete = false
|
||||
next.Resources[scope.ResourceID] = cloneRevisions(oldRevisions)
|
||||
items = append(items, failedResource(scope.ResourceID, err))
|
||||
continue
|
||||
}
|
||||
|
||||
// Skipped nodes are deterministic: this connector has no ingest path for
|
||||
// them, so they are reported once and never retried — unlike a failed
|
||||
// read below, which must stay retryable.
|
||||
for _, node := range skipped {
|
||||
logger.Infof(ctx, "[DingTalk] skip node %s (name=%q type=%s category=%s extension=%s): %s",
|
||||
node.ID, node.title(), node.Type, node.Category, node.Extension, skipReason(node))
|
||||
}
|
||||
|
||||
newRevisions := make(map[string]string, len(documents))
|
||||
currentDocuments := make(map[string]struct{}, len(documents))
|
||||
synced := 0
|
||||
failed := 0
|
||||
|
||||
for _, document := range documents {
|
||||
if document.ID == "" {
|
||||
|
|
@ -465,10 +544,13 @@ func (c *Connector) sync(
|
|||
if isContextError(err) {
|
||||
return nil, nil, err
|
||||
}
|
||||
logger.Warnf(ctx,
|
||||
"[DingTalk] read document %s (name=%q extension=%s) failed, will retry next sync: %v",
|
||||
document.ID, document.title(), document.Extension, err)
|
||||
items = append(items, failedDocument(
|
||||
scope.ResourceID, scope.Reference.WorkspaceID, document, err,
|
||||
))
|
||||
failedDocuments++
|
||||
failed++
|
||||
if existed {
|
||||
// Do not advance failed documents. The next incremental run
|
||||
// must retry them even if modifiedTime remains unchanged.
|
||||
|
|
@ -480,6 +562,7 @@ func (c *Connector) sync(
|
|||
items = append(items, fetchedDocument(
|
||||
scope.ResourceID, scope.Reference.WorkspaceID, document, rendered,
|
||||
))
|
||||
synced++
|
||||
newRevisions[document.ID] = revision
|
||||
}
|
||||
|
||||
|
|
@ -492,6 +575,14 @@ func (c *Connector) sync(
|
|||
}
|
||||
}
|
||||
next.Resources[scope.ResourceID] = newRevisions
|
||||
|
||||
// One line per scope, in the same shape as the IMA connector's. total
|
||||
// counts every file the scope listed — synced + still-unchanged +
|
||||
// failed + skipped — so a scope that quietly loses two thirds of its
|
||||
// nodes can no longer look like a clean run.
|
||||
logger.Infof(ctx, "[DingTalk] scope %s: total=%d synced=%d skipped=%d failed=%d",
|
||||
scopeLabel(scope), len(documents)+len(skipped), synced, len(skipped), failed)
|
||||
failedDocuments += failed
|
||||
}
|
||||
|
||||
// Reconcile the union of all selections. Moving a document between two
|
||||
|
|
@ -670,27 +761,34 @@ func childByID(children []node, nodeID string) (node, bool) {
|
|||
return node{}, false
|
||||
}
|
||||
|
||||
func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, error) {
|
||||
// scanScope lists every ingestible document in one scope, together with the
|
||||
// nodes it saw but cannot ingest. The skipped nodes are returned rather than
|
||||
// discarded so the caller can report them: a full sync must never look clean
|
||||
// while silently dropping part of the tree.
|
||||
func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, []node, error) {
|
||||
if scope.Document != nil {
|
||||
return []node{*scope.Document}, nil
|
||||
return []node{*scope.Document}, nil, nil
|
||||
}
|
||||
return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID)
|
||||
}
|
||||
|
||||
// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal
|
||||
// only; every file is either an ingestible document or a skip.
|
||||
func scanWorkspace(
|
||||
ctx context.Context,
|
||||
api dingTalkAPI,
|
||||
workspaceID string,
|
||||
rootNodeID string,
|
||||
) ([]node, error) {
|
||||
) ([]node, []node, error) {
|
||||
queue := []string{rootNodeID}
|
||||
visitedParents := make(map[string]struct{})
|
||||
seenNodes := make(map[string]struct{})
|
||||
var documents []node
|
||||
var skipped []node
|
||||
|
||||
for len(queue) > 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
parentID := queue[0]
|
||||
queue = queue[1:]
|
||||
|
|
@ -701,24 +799,30 @@ func scanWorkspace(
|
|||
|
||||
children, err := api.listNodes(ctx, parentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, child := range children {
|
||||
if child.ID == "" {
|
||||
continue
|
||||
}
|
||||
if child.WorkspaceID != "" && child.WorkspaceID != workspaceID {
|
||||
return nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)
|
||||
return nil, nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)
|
||||
}
|
||||
if _, seen := seenNodes[child.ID]; seen {
|
||||
continue
|
||||
}
|
||||
seenNodes[child.ID] = struct{}{}
|
||||
if len(seenNodes) > maxTraversalNodes {
|
||||
return nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)
|
||||
return nil, nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)
|
||||
}
|
||||
if child.isDocument() {
|
||||
switch {
|
||||
case child.isDocument():
|
||||
documents = append(documents, child)
|
||||
case child.isFolder():
|
||||
// Containers hold no content of their own, so they are never
|
||||
// reported as skipped.
|
||||
default:
|
||||
skipped = append(skipped, child)
|
||||
}
|
||||
if child.isFolder() || child.HasChildren {
|
||||
queue = append(queue, child.ID)
|
||||
|
|
@ -728,7 +832,10 @@ func scanWorkspace(
|
|||
sort.SliceStable(documents, func(i, j int) bool {
|
||||
return documents[i].ID < documents[j].ID
|
||||
})
|
||||
return documents, nil
|
||||
sort.SliceStable(skipped, func(i, j int) bool {
|
||||
return skipped[i].ID < skipped[j].ID
|
||||
})
|
||||
return documents, skipped, nil
|
||||
}
|
||||
|
||||
func cloneRevisions(revisions map[string]string) map[string]string {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package dingtalk
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
|
@ -8,6 +9,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
|
|
@ -110,10 +112,14 @@ func TestConnectorListsWorkspacesAndFetchesNestedDocuments(t *testing.T) {
|
|||
if err != nil || len(documents) != 1 || documents[0].Type != "document" {
|
||||
t.Fatalf("folder children = %#v, %v; want one document", documents, err)
|
||||
}
|
||||
if _, err := connector.ListResources(
|
||||
// A document is a leaf: expanding it is answered with no children rather
|
||||
// than an error, so a picker expansion cannot turn a selection that syncs
|
||||
// fine into a failure.
|
||||
leafChildren, err := connector.ListResources(
|
||||
context.Background(), testConfig(), documents[0].ExternalID,
|
||||
); !errors.Is(err, datasource.ErrResourceNotFound) {
|
||||
t.Fatalf("expanding a document error = %v, want ErrResourceNotFound", err)
|
||||
)
|
||||
if err != nil || len(leafChildren) != 0 {
|
||||
t.Fatalf("expanding a document = %#v, %v; want no children", leafChildren, err)
|
||||
}
|
||||
ancestors, err := connector.ResolveResourceAncestors(
|
||||
context.Background(), testConfig(), []string{documents[0].ExternalID},
|
||||
|
|
@ -441,3 +447,323 @@ func TestValidateProbesNodeAndDocumentAccess(t *testing.T) {
|
|||
t.Fatal("Validate() error = nil, want node access failure")
|
||||
}
|
||||
}
|
||||
|
||||
// Expanding a document is answered with no children rather than an error: a
|
||||
// leaf genuinely has none, and the picker renders an error as a failed
|
||||
// expansion — a toast on a step whose selection syncs perfectly well. The
|
||||
// listing already reports HasChildren=false for a document, which keeps the
|
||||
// picker from offering the expander; this is the answer for a client that
|
||||
// expands anyway, or for a selection saved by a build that did offer it.
|
||||
func TestListResourcesReportsNoChildrenForALeafDocument(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "team", RootNodeID: "team-root", Name: "Team"}},
|
||||
nodes: map[string][]node{
|
||||
"team-root": {{
|
||||
ID: "doc-1", WorkspaceID: "team", Name: "Plan.adoc",
|
||||
Type: "FILE", Category: "ALIDOC", Extension: "adoc",
|
||||
}},
|
||||
},
|
||||
}
|
||||
documentID, err := encodeResourceReference(resourceReference{
|
||||
WorkspaceID: "team", NodeID: "doc-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resources, err := testConnector(api).ListResources(context.Background(), testConfig(), documentID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources(%q) error = %v, want an empty listing", documentID, err)
|
||||
}
|
||||
if len(resources) != 0 {
|
||||
t.Fatalf("ListResources(%q) = %#v, want no children", documentID, resources)
|
||||
}
|
||||
}
|
||||
|
||||
// captureLogs redirects the process logger into a buffer for one test. The sync
|
||||
// log is the surface a skipped node has to appear on, so it is asserted on
|
||||
// directly rather than through an internal counter.
|
||||
func captureLogs(t *testing.T) *bytes.Buffer {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
logger.SetOutput(&buffer)
|
||||
t.Cleanup(logger.ConfigureFromEnv)
|
||||
return &buffer
|
||||
}
|
||||
|
||||
// skippedNodesFixture mixes supported documents with the node types this
|
||||
// connector deliberately cannot ingest, spread over a folder tree: reporting
|
||||
// must survive nesting and must not disturb what is fetched.
|
||||
func skippedNodesFixture() *fakeAPI {
|
||||
return &fakeAPI{
|
||||
workspaces: []workspace{{ID: "space", RootNodeID: "root", Name: "Space"}},
|
||||
nodes: map[string][]node{
|
||||
"root": {
|
||||
{ID: "docs", WorkspaceID: "space", Name: "Docs", Type: "FOLDER"},
|
||||
{
|
||||
ID: "video-1", WorkspaceID: "space", Name: "Lesson.mp4", Type: "FILE",
|
||||
Category: "VIDEO", Extension: "mp4",
|
||||
},
|
||||
{
|
||||
ID: "table-1", WorkspaceID: "space", Name: "Roadmap.able", Type: "FILE",
|
||||
Category: "ALIDOC", Extension: "able",
|
||||
},
|
||||
},
|
||||
"docs": {
|
||||
{
|
||||
ID: "doc-1", WorkspaceID: "space", Name: "Runbook.adoc", Type: "FILE",
|
||||
Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r1",
|
||||
},
|
||||
{
|
||||
ID: "doc-2", WorkspaceID: "space", Name: "Policy.adoc", Type: "FILE",
|
||||
Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r2",
|
||||
},
|
||||
{
|
||||
ID: "mind-1", WorkspaceID: "space", Name: "Plan.amind", Type: "FILE",
|
||||
Category: "ALIDOC", Extension: "amind",
|
||||
},
|
||||
{ID: "empty", WorkspaceID: "space", Name: "Empty", Type: "FOLDER"},
|
||||
},
|
||||
"empty": nil,
|
||||
},
|
||||
blocks: map[string][]json.RawMessage{
|
||||
"doc-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"run"}}`)},
|
||||
"doc-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"policy"}}`)},
|
||||
},
|
||||
nodeErrors: make(map[string]error),
|
||||
blockErrors: make(map[string]error),
|
||||
}
|
||||
}
|
||||
|
||||
// Seeing an unsupported node must not change what is fetched: the supported
|
||||
// siblings still sync exactly as before, and nothing is requested for a skip.
|
||||
func TestFetchAllKeepsSyncingSupportedSiblingsOfSkippedNodes(t *testing.T) {
|
||||
api := skippedNodesFixture()
|
||||
items, err := testConnector(api).FetchAll(
|
||||
context.Background(), testConfig("space"), []string{"space"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("FetchAll() returned %d items, want 2: %#v", len(items), items)
|
||||
}
|
||||
byID := make(map[string]types.FetchedItem, len(items))
|
||||
for _, item := range items {
|
||||
byID[item.ExternalID] = item
|
||||
}
|
||||
if len(byID) != 2 || byID["doc-1"].ContentType != "text/markdown" ||
|
||||
byID["doc-2"].ContentType != "text/markdown" {
|
||||
t.Fatalf("supported documents = %#v", byID)
|
||||
}
|
||||
for _, skipped := range []string{"video-1", "table-1", "mind-1"} {
|
||||
if _, exists := byID[skipped]; exists {
|
||||
t.Fatalf("unsupported node %q was synced: %#v", skipped, byID[skipped])
|
||||
}
|
||||
if api.blockCalls[skipped] != 0 {
|
||||
t.Fatalf("unsupported node %q was requested: %#v", skipped, api.blockCalls)
|
||||
}
|
||||
}
|
||||
if len(api.blockCalls) != 2 || api.blockCalls["doc-1"] != 1 || api.blockCalls["doc-2"] != 1 {
|
||||
t.Fatalf("block calls = %#v, want one per supported document", api.blockCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// Every node the connector drops must reach the sync log with its identity and
|
||||
// the concrete reason it cannot be ingested: media is never downloaded on
|
||||
// purpose, while a native DingTalk type simply has no ingest path yet.
|
||||
func TestSyncLogsEverySkippedNodeWithItsReason(t *testing.T) {
|
||||
api := skippedNodesFixture()
|
||||
logs := captureLogs(t)
|
||||
if _, err := testConnector(api).FetchAll(
|
||||
context.Background(), testConfig("space"), []string{"space"},
|
||||
); err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
|
||||
output := logs.String()
|
||||
for _, want := range []string{
|
||||
`[DingTalk] skip node video-1 (name="Lesson.mp4" type=FILE category=VIDEO extension=mp4): ` +
|
||||
"video/media files are deliberately not downloaded by this connector",
|
||||
`[DingTalk] skip node table-1 (name="Roadmap.able" type=FILE category=ALIDOC extension=able): ` +
|
||||
"DingTalk multi-dimensional table has no ingest path in this connector yet",
|
||||
`[DingTalk] skip node mind-1 (name="Plan.amind" type=FILE category=ALIDOC extension=amind): ` +
|
||||
"DingTalk mind map has no ingest path in this connector yet",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("sync log is missing:\n%s\ngot:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
for _, folder := range []string{"skip node docs", "skip node empty"} {
|
||||
if strings.Contains(output, folder) {
|
||||
t.Fatalf("folder reported as a skipped node: %q in\n%s", folder, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The per-scope summary states how much of the scope arrived, in the same shape
|
||||
// as the IMA connector's per-knowledge-base line.
|
||||
func TestSyncLogsPerScopeSummary(t *testing.T) {
|
||||
api := skippedNodesFixture()
|
||||
logs := captureLogs(t)
|
||||
if _, err := testConnector(api).FetchAll(
|
||||
context.Background(), testConfig("space"), []string{"space"},
|
||||
); err != nil {
|
||||
t.Fatalf("FetchAll() error = %v", err)
|
||||
}
|
||||
want := "[DingTalk] scope space/root: total=5 synced=2 skipped=3 failed=0"
|
||||
if !strings.Contains(logs.String(), want) {
|
||||
t.Fatalf("sync log is missing %q:\n%s", want, logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A transient read failure is counted as failed — not as skipped — is logged as
|
||||
// retryable, and still leaves the document out of the cursor.
|
||||
func TestSyncSummaryCountsTransientFailuresAndStillRetries(t *testing.T) {
|
||||
api := skippedNodesFixture()
|
||||
api.blockErrors = map[string]error{"doc-2": errors.New("storage temporarily unavailable")}
|
||||
logs := captureLogs(t)
|
||||
|
||||
items, err := testConnector(api).FetchAll(
|
||||
context.Background(), testConfig("space"), []string{"space"},
|
||||
)
|
||||
var partial *datasource.PartialFetchError
|
||||
if !errors.As(err, &partial) {
|
||||
t.Fatalf("FetchAll() error = %v, want PartialFetchError", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("FetchAll() items = %#v, want 1 document and 1 failure", items)
|
||||
}
|
||||
output := logs.String()
|
||||
if !strings.Contains(output, "storage temporarily unavailable") ||
|
||||
!strings.Contains(output, "failed, will retry next sync") {
|
||||
t.Fatalf("transient failure was not logged as retryable:\n%s", output)
|
||||
}
|
||||
want := "[DingTalk] scope space/root: total=5 synced=1 skipped=3 failed=1"
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("sync log is missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
|
||||
// Containers are not content: a tree of folders alone must produce no skip
|
||||
// report at all, only a zeroed summary.
|
||||
func TestSyncLogsNoSkipsForFolderOnlyTree(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "space", RootNodeID: "root"}},
|
||||
nodes: map[string][]node{
|
||||
"root": {{ID: "outer", WorkspaceID: "space", Name: "Outer", Type: "FOLDER"}},
|
||||
"outer": {{ID: "inner", WorkspaceID: "space", Name: "Inner", Type: "FOLDER"}},
|
||||
"inner": nil,
|
||||
},
|
||||
}
|
||||
logs := captureLogs(t)
|
||||
items, err := testConnector(api).FetchAll(
|
||||
context.Background(), testConfig("space"), []string{"space"},
|
||||
)
|
||||
if err != nil || len(items) != 0 {
|
||||
t.Fatalf("FetchAll() = %#v, %v; want no items", items, err)
|
||||
}
|
||||
output := logs.String()
|
||||
if strings.Contains(output, "skip node") {
|
||||
t.Fatalf("folder-only tree reported skips:\n%s", output)
|
||||
}
|
||||
want := "[DingTalk] scope space/root: total=0 synced=0 skipped=0 failed=0"
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("sync log is missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
|
||||
// scanScope hands the caller the skipped nodes with their DingTalk identity
|
||||
// intact, and a single-document scope has no tree to skip anything from.
|
||||
func TestScanScopeReportsSkippedNodesAndSingleDocumentScopeHasNone(t *testing.T) {
|
||||
api := skippedNodesFixture()
|
||||
documents, skipped, err := scanScope(context.Background(), api, syncScope{
|
||||
ResourceID: "resource",
|
||||
Reference: resourceReference{WorkspaceID: "space"},
|
||||
StartNodeID: "root",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("scanScope() error = %v", err)
|
||||
}
|
||||
if len(documents) != 2 {
|
||||
t.Fatalf("scanScope() documents = %#v, want 2", documents)
|
||||
}
|
||||
if len(skipped) != 3 {
|
||||
t.Fatalf("scanScope() skipped = %#v, want 3", skipped)
|
||||
}
|
||||
got := make(map[string]node, len(skipped))
|
||||
for _, item := range skipped {
|
||||
got[item.ID] = item
|
||||
}
|
||||
if got["video-1"].Category != "VIDEO" || got["video-1"].Extension != "mp4" ||
|
||||
got["video-1"].Name != "Lesson.mp4" ||
|
||||
got["table-1"].Category != "ALIDOC" || got["table-1"].Extension != "able" ||
|
||||
got["mind-1"].Extension != "amind" {
|
||||
t.Fatalf("skipped nodes lost their identity: %#v", got)
|
||||
}
|
||||
|
||||
document := syncDocument()
|
||||
documents, skipped, err = scanScope(context.Background(), api, syncScope{
|
||||
ResourceID: "resource",
|
||||
Reference: resourceReference{WorkspaceID: "space"},
|
||||
Document: &document,
|
||||
})
|
||||
if err != nil || len(documents) != 1 || documents[0].ID != "doc" || len(skipped) != 0 {
|
||||
t.Fatalf("single-document scanScope() = %#v, %#v, %v", documents, skipped, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The reason has to say which kind of unsupported a node is: a video is skipped
|
||||
// on purpose, a native type has simply not been implemented.
|
||||
func TestSkipReasonDistinguishesMediaFromUnimplementedTypes(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
label string
|
||||
node node
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"video category",
|
||||
node{Type: "FILE", Category: "VIDEO", Extension: "mp4"},
|
||||
"video/media files are deliberately not downloaded by this connector",
|
||||
},
|
||||
{
|
||||
"video extension without a category",
|
||||
node{Type: "FILE", Category: "OTHER", Extension: "MOV"},
|
||||
"video/media files are deliberately not downloaded by this connector",
|
||||
},
|
||||
{
|
||||
"audio extension",
|
||||
node{Type: "FILE", Category: "OTHER", Extension: "mp3"},
|
||||
"video/media files are deliberately not downloaded by this connector",
|
||||
},
|
||||
{
|
||||
// Only the types with a name worth printing are in the map; every
|
||||
// other unsupported node gets the generic reason. Kept as a
|
||||
// regression guard on the map.
|
||||
"type without a dedicated label",
|
||||
node{Type: "FILE", Category: "ALIDOC", Extension: "axls"},
|
||||
"no ingest path for this DingTalk node type in this connector yet",
|
||||
},
|
||||
{
|
||||
"multidimensional table",
|
||||
node{Type: "FILE", Category: "ALIDOC", Extension: "able"},
|
||||
"DingTalk multi-dimensional table has no ingest path in this connector yet",
|
||||
},
|
||||
{
|
||||
"mind map",
|
||||
node{Type: "FILE", Category: "ALIDOC", Extension: "amind"},
|
||||
"DingTalk mind map has no ingest path in this connector yet",
|
||||
},
|
||||
{
|
||||
"unknown type",
|
||||
node{Type: "FILE", Category: "OTHER", Extension: "bin"},
|
||||
"no ingest path for this DingTalk node type in this connector yet",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
if got := skipReason(testCase.node); got != testCase.want {
|
||||
t.Fatalf("skipReason() = %q, want %q", got, testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue