diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.go index d3a77d33..4bb0a3d0 100644 --- a/internal/datasource/connector/dingtalk/connector.go +++ b/internal/datasource/connector/dingtalk/connector.go @@ -116,9 +116,10 @@ const maxValidateProbes = 5 // // The operator is not expected to reach every workspace in the tenant: app // credentials are valid as long as one reachable workspace yields one readable -// document. Reading the first document of the first workspace and failing on it -// rejects a working configuration whenever an unrelated workspace — one the -// operator was never granted — happens to be listed first. +// document. A failed probe does not condemn that workspace; the walk keeps +// looking, inside the probe cap, for another document the operator can read. +// When the cap itself stops a conclusive walk, the error says how many +// documents were read. func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSourceConfig) error { cfg, err := parseConfig(dataSourceConfig) if err != nil { @@ -138,14 +139,16 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo workspaceFail int probes int listings int + probeCapHit bool ) - for _, item := range workspaces { + for i, item := range workspaces { rootNodeID := strings.TrimSpace(item.RootNodeID) if rootNodeID == "" { continue } - document, found, unexplored, err := firstValidateDocument( - ctx, api, rootNodeID, settings, &listings, + readable, unexplored, hitCap, err := sampleWorkspace( + ctx, api, item.Name, rootNodeID, settings, + &listings, &probes, &sawDocument, &lastErr, ) if err != nil { lastErr = fmt.Errorf("workspace %q: %w", item.Name, err) @@ -157,29 +160,24 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo workspaceFail++ continue } + if readable { + return nil + } if unexplored { inconclusive = true } - if !found { - continue - } - sawDocument = true - probes++ - if err := verifyDocument(ctx, api, document); err != nil { - lastErr = fmt.Errorf("workspace %q document %q: %w", item.Name, document.Name, err) - if ctx.Err() != nil || isContextError(err) { - return fmt.Errorf("validate DingTalk data source: %w", lastErr) - } - } else { - return nil - } + // Once the cap is spent no further probe can succeed, so walking on + // could only find an unlistable folder and turn the failed probes + // into acceptance. if probes >= maxValidateProbes { + probeCapHit = hitCap || i+1 < len(workspaces) break } } // Folders were left unexplored, because the listing budget ran out or a // folder could not be listed, so an unreadable document elsewhere does not - // prove the data source is unusable. + // prove the data source is unusable. A probe cap hit in that situation + // stays quiet too: the walk cannot claim it sampled every visible document. if inconclusive { return nil } @@ -194,6 +192,12 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo if lastErr == nil { return nil } + if probeCapHit { + return fmt.Errorf( + "validate DingTalk data source: %w; only the first %d documents were probed", + lastErr, maxValidateProbes, + ) + } return fmt.Errorf("validate DingTalk data source: %w", lastErr) } @@ -205,45 +209,83 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo // client's retries of each). const maxValidateListings = 20 -// firstValidateDocument walks a workspace breadth-first, the way scanWorkspace -// does, and returns the first document it finds. It requests one page at a -// time instead of listing whole folders, so the budget bounds real requests. -// unexplored reports that some nodes were never seen, because the budget ran -// out or a folder could not be listed, so the walk cannot prove the workspace -// holds no readable document. +// sampleWorkspace walks one workspace breadth-first, the way scanWorkspace +// does, and probes ingestible documents until one is readable, the probe cap +// is spent, or the walk runs out of nodes. It requests one page at a time +// instead of listing whole folders, so the budget bounds real requests. +// +// unexplored reports that some folder was never opened or could not be listed, +// so a failed probe elsewhere must not reject the data source. readable reports +// that a probe succeeded. The walk stops as soon as the failed probe that +// spends maxValidateProbes returns, without another listing; capHit then +// reports that nodes of this workspace were still left to visit. The caller +// turns that into an error only when the walk is conclusive. +// +// A folder's later pages are held back once the listings still in the budget +// are no longer enough to open every sibling already queued, so each of those +// siblings still gets one page. The unread tail is not itself "unexplored" +// when a document was probed after it was skipped: the reservation is what +// made room for that probe. A tail skipped with no probe after it stays +// inconclusive, because the only readable document may sit on a page the +// sample never requested. A folder that receives no page at all, a subfolder +// listing error, a repeated page token, or a tail with no sibling waiting all +// stay inconclusive too. +// +// Any failed page of the workspace root is a hard error, matching listNodes +// during sync. Subfolder listing errors stay inconclusive unless the request +// was cancelled or timed out. // // The walk selects nodes with the same settings-aware predicate the picker and // the sync use, so a type the data source does not ingest is never probed. -func firstValidateDocument( +func sampleWorkspace( ctx context.Context, api dingTalkAPI, + workspaceName string, rootNodeID string, settings documentSettings, listings *int, -) (document node, found, unexplored bool, err error) { + probes *int, + sawDocument *bool, + lastErr *error, +) (readable, unexplored, capHit bool, err error) { queue := []string{rootNodeID} visited := map[string]struct{}{rootNodeID: {}} + uncoveredSkip := false for len(queue) > 0 { parent := queue[0] queue = queue[1:] // Finish every page of a folder before descending, like listNodes // does for scanWorkspace: a document on a later page of this folder // must be found before the budget is spent on its subfolders. + // Siblings already on the queue stay ahead of those subfolders. var subfolders []string pageToken := "" seenTokens := map[string]struct{}{} for { + // Reserve one listing per sibling already queued before spending + // another page on this folder. The root's own pages have an empty + // queue here — its children are not queued until the root is + // finished — so a document on the root's next page is still read + // before the walk descends. + if pageToken != "" && *listings+len(queue) >= maxValidateListings { + if len(queue) == 0 { + unexplored = true + } else { + uncoveredSkip = true + } + break + } rootPage := parent == rootNodeID && pageToken == "" if !rootPage { if *listings >= maxValidateListings { - return node{}, false, true, nil + return false, true, false, nil } *listings++ } children, next, listErr := api.listNodesPage(ctx, parent, pageToken) if listErr != nil { - if rootPage || ctx.Err() != nil || isContextError(listErr) { - return node{}, false, false, listErr + if parent == rootNodeID || ctx.Err() != nil || isContextError(listErr) { + return false, false, false, listErr } // The documents this operator can read may live in exactly // this folder, so its failure leaves the walk inconclusive @@ -251,9 +293,27 @@ func firstValidateDocument( unexplored = true break } - for _, child := range children { + for i, child := range children { if child.isIngestible(settings) { - return child, true, unexplored, nil + *probes++ + *sawDocument = true + uncoveredSkip = false + probeErr := verifyDocument(ctx, api, child) + if probeErr != nil { + if ctx.Err() != nil || isContextError(probeErr) { + return false, false, false, fmt.Errorf("document %q: %w", child.Name, probeErr) + } + *lastErr = fmt.Errorf( + "workspace %q document %q: %w", workspaceName, child.Name, probeErr, + ) + if *probes >= maxValidateProbes { + more := i+1 < len(children) || len(subfolders) > 0 || + next != "" || len(queue) > 0 + return false, unexplored, more, nil + } + continue + } + return true, false, false, nil } if child.isFolder() || child.HasChildren { if _, seen := visited[child.ID]; seen { @@ -277,7 +337,10 @@ func firstValidateDocument( } queue = append(queue, subfolders...) } - return node{}, false, unexplored, nil + if uncoveredSkip { + unexplored = true + } + return false, unexplored, false, nil } // verifyDocument proves one visible document is readable. Native documents are diff --git a/internal/datasource/connector/dingtalk/validate_test.go b/internal/datasource/connector/dingtalk/validate_test.go index 86ac79c6..02121e3c 100644 --- a/internal/datasource/connector/dingtalk/validate_test.go +++ b/internal/datasource/connector/dingtalk/validate_test.go @@ -147,6 +147,11 @@ func TestValidateReportsWhenNoVisibleDocumentIsReadable(t *testing.T) { t.Fatalf("Validate error should carry %s, got: %v", want, err) } } + // Both documents were probed, so the cap did not stop the walk and the + // error must not claim that it did. + if strings.Contains(err.Error(), "only the first") { + t.Fatalf("Validate error should not mention the probe cap, got: %v", err) + } } // A tenant with no documents anywhere leaves nothing to read, so the @@ -194,9 +199,10 @@ func TestValidateSurvivesWorkspaceListingFailure(t *testing.T) { } } -// An app without the document read permission fails every probe. Validate must -// probe at most one document per workspace and stop after maxValidateProbes, -// instead of firing one call per document across the whole tenant. +// An app without the document read permission fails every probe. Validate keeps +// going to the next document in the same workspace, but it still stops after +// maxValidateProbes reads and says so, instead of firing one call per document +// across the whole tenant. func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) { api := &fakeAPI{ nodes: map[string][]node{}, @@ -216,20 +222,32 @@ func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) { } c := testConnector(api) - if err := c.Validate(context.Background(), testConfig()); err == nil { + err := c.Validate(context.Background(), testConfig()) + if err == nil { t.Fatal("Validate must fail when no visible document can be read") } total := 0 for _, n := range api.blockCalls { total += n } - if total > maxValidateProbes { - t.Fatalf("Validate made %d document probes, want at most %d", total, maxValidateProbes) + if total != maxValidateProbes { + t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes) } - for w := 0; w < workspaces; w++ { - if api.blockCalls[fmt.Sprintf("doc-%d-1", w)] != 0 { - t.Fatalf("workspace root-%d was probed past its first document", w) - } + // The first workspace has four unreadable documents, so the cap must be + // spent there and on the next workspace's first document — not one probe + // per workspace, and not on a third workspace. + if api.blockCalls["doc-0-1"] != 1 { + t.Fatal("a failed probe must not stop the workspace from trying its next document") + } + if api.blockCalls["doc-1-0"] != 1 || api.blockCalls["doc-1-1"] != 0 || api.blockCalls["doc-2-0"] != 0 { + t.Fatalf("probe cap was not applied across workspaces, calls=%v", api.blockCalls) + } + wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes) + if !strings.Contains(err.Error(), wantNote) { + t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err) + } + if !strings.Contains(err.Error(), `document "doc-1-0"`) { + t.Fatalf("Validate error should name the last probed document, got: %v", err) } } @@ -554,10 +572,14 @@ func TestValidateBudgetCountsNodeListingPages(t *testing.T) { // A folder's later pages must be read before its subfolders: otherwise a root // whose first page is all folders spends the budget below them and never sees -// the document on the root's second page. +// the document on the root's second page. The document is unreadable and the +// twenty subfolders cannot all be listed after that page spends a budget unit, +// so the rest of the tree is unknown and Validate accepts. func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) { var mu sync.Mutex probed := false + folderLists := 0 + folderListsBeforeProbe := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { @@ -578,6 +600,12 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) { _, _ = w.Write([]byte( `{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) default: + mu.Lock() + folderLists++ + if !probed { + folderListsBeforeProbe++ + } + mu.Unlock() _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`)) } case strings.HasPrefix(r.URL.Path, "/v1.0/doc/suites/documents/"): @@ -598,8 +626,14 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) { if !probed { t.Fatalf("the document on the root's second page was never probed (err=%v)", err) } - if err == nil { - t.Fatal("Validate must report the unreadable document instead of accepting") + if folderListsBeforeProbe != 0 { + t.Fatalf("listed %d subfolders before probing the root's second page", folderListsBeforeProbe) + } + if folderLists == 0 { + t.Fatal("Validate stopped at the unreadable document instead of continuing the walk") + } + if err != nil { + t.Fatalf("Validate must accept once the listing budget leaves folders unexplored, got: %v", err) } } @@ -640,3 +674,530 @@ func TestValidateStopsOnRepeatedPageToken(t *testing.T) { t.Fatalf("folder was listed %d times, want 2 (first page + the repeated token once)", folderRequests) } } + +// The first document of a workspace can be unreadable while a later one in the +// same folder is not. Stopping at the first probe would reject a data source +// the operator can actually sync. +func TestValidateTriesTheNextDocumentAfterAFailedProbe(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}}, + nodes: map[string][]node{ + "root-a": { + {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + }, + }, + blocks: map[string][]json.RawMessage{ + "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)}, + }, + blockErrors: map[string]error{ + "doc-bad": errors.New("forbidden.accessDenied: the operator has no permission"), + }, + } + + if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept once a later document in the folder is readable, got: %v", err) + } + if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 { + t.Fatalf("both documents should be probed, calls=%v", api.blockCalls) + } +} + +// A readable document that lives in a later sibling folder must still be +// reached after the first folder's document fails. The same walk, when every +// document fails, reports the later one: the workspace is conclusive. +func TestValidateTriesADocumentInALaterSiblingFolder(t *testing.T) { + denied := errors.New("forbidden.accessDenied: the operator has no permission") + t.Run("later folder readable", func(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}}, + nodes: map[string][]node{ + "root-a": { + {ID: "folder-a", Name: "A", Type: "FOLDER"}, + {ID: "folder-b", Name: "B", Type: "FOLDER"}, + }, + "folder-a": { + {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + }, + "folder-b": { + {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + }, + }, + blocks: map[string][]json.RawMessage{ + "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)}, + }, + blockErrors: map[string]error{"doc-bad": denied}, + } + if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept the readable document in the later folder, got: %v", err) + } + if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 { + t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls) + } + }) + t.Run("every document unreadable", func(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}}, + nodes: map[string][]node{ + "root-a": { + {ID: "folder-a", Name: "A", Type: "FOLDER"}, + {ID: "folder-b", Name: "B", Type: "FOLDER"}, + }, + "folder-a": { + {ID: "doc-first", Name: "First", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + }, + "folder-b": { + {ID: "doc-second", Name: "Second", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}, + }, + }, + blockErrors: map[string]error{"doc-first": denied, "doc-second": denied}, + } + err := testConnector(api).Validate(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), `document "Second"`) { + t.Fatalf("Validate must report the later unreadable document, got: %v", err) + } + if api.blockCalls["doc-first"] != 1 || api.blockCalls["doc-second"] != 1 { + t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls) + } + }) +} + +// An unreadable document on a later root page is conclusive when the rest of +// the workspace fits in the listing budget. The page is still read before any +// subfolder, and the walk then finishes those subfolders instead of stopping +// on the first failed probe. +func TestValidateReportsUnreadableDocumentOnALaterRootPage(t *testing.T) { + var mu sync.Mutex + probed := false + folderLists := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1.0/oauth2/accessToken": + _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`)) + case "/v2.0/wiki/workspaces": + _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`)) + case "/v2.0/wiki/nodes": + parent := r.URL.Query().Get("parentNodeId") + switch { + case parent == "root-a" && r.URL.Query().Get("nextToken") == "": + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`)) + case parent == "root-a": + _, _ = w.Write([]byte( + `{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) + default: + mu.Lock() + folderLists++ + mu.Unlock() + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`)) + } + case "/v1.0/doc/suites/documents/doc/blocks": + mu.Lock() + probed = true + mu.Unlock() + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + err := testConnector(testClient(server)).Validate(context.Background(), testConfig()) + mu.Lock() + defer mu.Unlock() + if !probed || folderLists != 1 { + t.Fatalf("probed=%v folderLists=%d, want the page probed and its subfolder listed", probed, folderLists) + } + if err == nil || !strings.Contains(err.Error(), `document "Doc"`) { + t.Fatalf("Validate must report the unreadable document, got: %v", err) + } + if strings.Contains(err.Error(), "only the first") { + t.Fatalf("one probed document must not be described as the probe cap, got: %v", err) + } +} + +// A root listing that fails on a later page is the same failure sync sees from +// listNodes: the workspace is not "unexplored", even when an earlier page +// already named a folder that holds a readable document. Another workspace can +// still prove the credentials, as a failed first page already can. +func TestValidateRejectsAFailedLaterRootPage(t *testing.T) { + const workspaces = `{"workspaces":[` + + `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` + + `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}` + handler := func(extraWorkspace bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1.0/oauth2/accessToken": + _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`)) + case "/v2.0/wiki/workspaces": + if extraWorkspace { + _, _ = w.Write([]byte(workspaces)) + return + } + _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`)) + case "/v2.0/wiki/nodes": + parent := r.URL.Query().Get("parentNodeId") + switch { + case parent == "root-a" && r.URL.Query().Get("nextToken") == "": + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`)) + case parent == "root-a": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"invalidRequest","message":"bad page"}`)) + case parent == "folder": + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"hidden","name":"Hidden",` + + `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) + default: + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Handbook",` + + `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) + } + case "/v1.0/doc/suites/documents/hidden/blocks", "/v1.0/doc/suites/documents/doc-b/blocks": + _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`)) + default: + http.NotFound(w, r) + } + } + } + t.Run("only workspace", func(t *testing.T) { + server := httptest.NewServer(handler(false)) + defer server.Close() + err := testConnector(testClient(server)).Validate(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), "status=400") || + !strings.Contains(err.Error(), `workspace "A"`) { + t.Fatalf("Validate must surface the root page failure, got: %v", err) + } + }) + t.Run("later workspace readable", func(t *testing.T) { + server := httptest.NewServer(handler(true)) + defer server.Close() + if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must survive one workspace whose later root page fails, got: %v", err) + } + }) +} + +// The first folder under the root can hold more non-document pages than the +// listing budget. Those pages must not be spent to the end while a sibling +// folder is still queued: the sibling is opened, and a document there is +// probed. A short folder is still finished first, so a document on its second +// page is not skipped just because a sibling is waiting. +func TestValidateOpensSiblingFolderBeforeSpendingTheListingBudget(t *testing.T) { + const fatPages = 30 + type listingStats struct { + fatPages int + probed map[string]int + } + start := func(t *testing.T, siblingReadable, docOnSecondPage bool) (*httptest.Server, *listingStats) { + t.Helper() + stats := &listingStats{probed: map[string]int{}} + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1.0/oauth2/accessToken": + _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`)) + case "/v2.0/wiki/workspaces": + _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`)) + case "/v2.0/wiki/nodes": + parent := r.URL.Query().Get("parentNodeId") + token := r.URL.Query().Get("nextToken") + switch parent { + case "root-a": + _, _ = w.Write([]byte(`{"nodes":[` + + `{"nodeId":"folder-a","type":"FOLDER"},` + + `{"nodeId":"folder-b","type":"FOLDER"}]}`)) + case "folder-a": + mu.Lock() + stats.fatPages++ + page := stats.fatPages + mu.Unlock() + if token != "" && token != strconv.Itoa(page-1) { + t.Errorf("folder-a page token = %q, previous page was %d", token, page-1) + } + item := fmt.Sprintf( + `{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}`, page, + ) + if docOnSecondPage && page == 2 { + item = `{"nodeId":"doc-early","name":"Early",` + + `"type":"FILE","category":"ALIDOC","extension":"adoc"}` + } + next := "" + if page < fatPages { + next = strconv.Itoa(page) + } + _, _ = fmt.Fprintf(w, `{"nodes":[%s],"nextToken":%q}`, item, next) + case "folder-b": + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Sibling",` + + `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) + default: + http.NotFound(w, r) + } + default: + const prefix = "/v1.0/doc/suites/documents/" + if !strings.HasPrefix(r.URL.Path, prefix) || !strings.HasSuffix(r.URL.Path, "/blocks") { + http.NotFound(w, r) + return + } + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), "/blocks") + mu.Lock() + stats.probed[id]++ + mu.Unlock() + if id == "doc-b" && !siblingReadable { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`)) + return + } + _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`)) + } + })) + return server, stats + } + + t.Run("sibling document readable", func(t *testing.T) { + server, stats := start(t, true, false) + defer server.Close() + if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept the sibling document, got: %v", err) + } + if stats.probed["doc-b"] != 1 { + t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages) + } + if stats.fatPages >= maxValidateListings { + t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits", + stats.fatPages, maxValidateListings) + } + }) + t.Run("sibling document unreadable", func(t *testing.T) { + server, stats := start(t, false, false) + defer server.Close() + err := testConnector(testClient(server)).Validate(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), `document "Sibling"`) { + t.Fatalf("Validate must report the sibling document, got: %v", err) + } + if stats.probed["doc-b"] != 1 { + t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages) + } + if stats.fatPages >= maxValidateListings { + t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits", + stats.fatPages, maxValidateListings) + } + }) + t.Run("short folder still yields its second page", func(t *testing.T) { + server, stats := start(t, false, true) + defer server.Close() + if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept the document on the short folder's second page, got: %v", err) + } + if stats.probed["doc-early"] != 1 || stats.probed["doc-b"] != 0 { + t.Fatalf("probes=%v, want only the document on page 2", stats.probed) + } + if stats.fatPages != 2 { + t.Fatalf("folder-a was listed %d times, want 2", stats.fatPages) + } + }) +} + +// Skipping the unread tail of a fat folder is what leaves room for its sibling. +// When that sample contains no document, the tail might still hold the only +// readable one, so an unreadable document in another workspace must not fail +// the data source. +func TestValidateSkippedTailWithoutAProbeStaysInconclusive(t *testing.T) { + var mu sync.Mutex + probed := false + fatPages := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1.0/oauth2/accessToken": + _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`)) + case "/v2.0/wiki/workspaces": + _, _ = w.Write([]byte(`{"workspaces":[` + + `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` + + `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`)) + case "/v2.0/wiki/nodes": + parent := r.URL.Query().Get("parentNodeId") + switch parent { + case "root-a": + _, _ = w.Write([]byte(`{"nodes":[` + + `{"nodeId":"folder-a","type":"FOLDER"},` + + `{"nodeId":"folder-b","type":"FOLDER"}]}`)) + case "folder-a": + mu.Lock() + fatPages++ + page := fatPages + mu.Unlock() + next := "" + if page < 30 { + next = strconv.Itoa(page) + } + _, _ = fmt.Fprintf(w, + `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`, + page, next) + case "folder-b": + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf-b","type":"FILE",` + + `"category":"FILE","extension":"pdf"}]}`)) + default: + _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Secret",` + + `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`)) + } + case "/v1.0/doc/suites/documents/doc-b/blocks": + mu.Lock() + probed = true + mu.Unlock() + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + err := testConnector(testClient(server)).Validate(context.Background(), testConfig()) + mu.Lock() + defer mu.Unlock() + if err != nil { + t.Fatalf("Validate must accept when the skipped tail was never probed, got: %v", err) + } + if !probed { + t.Fatal("the other workspace's document was never probed") + } + if fatPages >= maxValidateListings { + t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages) + } +} + +// An unlistable folder already makes the walk inconclusive. Hitting the probe +// cap afterwards must not turn that into a failure: a readable document may +// still sit in the folder that could not be listed. +func TestValidateProbeCapDoesNotOverrideAnInconclusiveWalk(t *testing.T) { + api := &fakeAPI{ + nodes: map[string][]node{ + "root-deep": {{ID: "folder", Name: "Folder", Type: "FOLDER"}}, + }, + nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")}, + blockErrors: map[string]error{}, + } + api.workspaces = []workspace{{ID: "deep", RootNodeID: "root-deep", Name: "Deep"}} + denied := errors.New("forbidden.accessDenied: the operator has no permission") + for w := 0; w < maxValidateProbes+2; w++ { + root := fmt.Sprintf("root-%d", w) + id := fmt.Sprintf("doc-%d", w) + api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root}) + api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}} + api.blockErrors[id] = denied + } + + if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept an inconclusive walk even after the probe cap, got: %v", err) + } + total := 0 + for _, n := range api.blockCalls { + total += n + } + if total != maxValidateProbes { + t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes) + } +} + +// Once the probe cap is spent no later probe can succeed. A workspace walked +// after that point must not get the chance to find an unlistable folder and +// turn every failed probe into acceptance. +func TestValidateStopsWalkingOnceTheProbeCapIsSpent(t *testing.T) { + api := &fakeAPI{ + nodes: map[string][]node{}, + nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")}, + blockErrors: map[string]error{}, + } + denied := errors.New("forbidden.accessDenied: the operator has no permission") + for w := 0; w < maxValidateProbes; w++ { + root := fmt.Sprintf("root-%d", w) + id := fmt.Sprintf("doc-%d", w) + api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root}) + api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}} + api.blockErrors[id] = denied + } + api.workspaces = append(api.workspaces, workspace{ID: "late", RootNodeID: "root-late", Name: "Late"}) + api.nodes["root-late"] = []node{{ID: "folder", Name: "Folder", Type: "FOLDER"}} + + err := testConnector(api).Validate(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), "no permission") { + t.Fatalf("Validate must report the failed probes, got: %v", err) + } + wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes) + if !strings.Contains(err.Error(), wantNote) { + t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err) + } +} + +// A tail skipped to make room for siblings is only covered by a probe made +// after it. A document probed before the skip says nothing about the skipped +// pages, so an empty sibling must leave the walk inconclusive. +func TestValidateSkippedTailIsNotCoveredByAnEarlierProbe(t *testing.T) { + var mu sync.Mutex + fatPages := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1.0/oauth2/accessToken": + _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`)) + case "/v2.0/wiki/workspaces": + _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`)) + case "/v2.0/wiki/nodes": + switch r.URL.Query().Get("parentNodeId") { + case "root-a": + _, _ = w.Write([]byte(`{"nodes":[` + + `{"nodeId":"doc-x","name":"X","type":"FILE","category":"ALIDOC","extension":"adoc"},` + + `{"nodeId":"folder-a","type":"FOLDER"},` + + `{"nodeId":"folder-b","type":"FOLDER"}]}`)) + case "folder-a": + mu.Lock() + fatPages++ + page := fatPages + mu.Unlock() + next := "" + if page < 30 { + next = strconv.Itoa(page) + } + _, _ = fmt.Fprintf(w, + `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`, + page, next) + case "folder-b": + _, _ = w.Write([]byte(`{"nodes":[]}`)) + default: + http.NotFound(w, r) + } + case "/v1.0/doc/suites/documents/doc-x/blocks": + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil { + t.Fatalf("Validate must accept when the skipped tail was never covered by a probe, got: %v", err) + } + mu.Lock() + defer mu.Unlock() + if fatPages >= maxValidateListings { + t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages) + } +} + +// A probe cancelled by the caller's deadline still names the document, so the +// operator can tell which read timed out. +func TestValidateContextErrorNamesTheProbedDocument(t *testing.T) { + api := &fakeAPI{ + workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}}, + nodes: map[string][]node{ + "root-a": {{ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}, + }, + blockErrors: map[string]error{"doc": context.DeadlineExceeded}, + } + err := testConnector(api).Validate(context.Background(), testConfig()) + if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), `document "Handbook"`) { + t.Fatalf("Validate must surface the timed-out probe with its document, got: %v", err) + } +}