fix(datasource/dingtalk): tighten Validate sampling edges (#4079)
* fix(datasource/dingtalk): tighten Validate sampling edges Keep probing later documents in the same workspace, hold listing budget for sibling folders that are already queued, fail closed when any root page cannot be listed, and say when the probe cap stops a conclusive walk. Fixes #4021 Co-authored-by: fight for you <wangrui19971019@users.noreply.github.com> * fix(datasource/dingtalk): stop Validate walk once probe cap is spent - Break the workspace walk as soon as maxValidateProbes is spent, so a later unlistable folder can no longer turn failed probes into acceptance. - Treat a skipped folder tail as covered only by a probe made after the skip, not by any earlier probe in the same workspace. - Keep the document name on probe context errors. --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: fight for you <wangrui19971019@users.noreply.github.com> Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
769a5aeca8
commit
57f8a65fec
|
|
@ -116,9 +116,10 @@ const maxValidateProbes = 5
|
|||
//
|
||||
// The operator is not expected to reach every workspace in the tenant: app
|
||||
// credentials are valid as long as one reachable workspace yields one readable
|
||||
// document. Reading the first document of the first workspace and failing on it
|
||||
// rejects a working configuration whenever an unrelated workspace — one the
|
||||
// operator was never granted — happens to be listed first.
|
||||
// document. A failed probe does not condemn that workspace; the walk keeps
|
||||
// looking, inside the probe cap, for another document the operator can read.
|
||||
// When the cap itself stops a conclusive walk, the error says how many
|
||||
// documents were read.
|
||||
func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSourceConfig) error {
|
||||
cfg, err := parseConfig(dataSourceConfig)
|
||||
if err != nil {
|
||||
|
|
@ -138,14 +139,16 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
|
|||
workspaceFail int
|
||||
probes int
|
||||
listings int
|
||||
probeCapHit bool
|
||||
)
|
||||
for _, item := range workspaces {
|
||||
for i, item := range workspaces {
|
||||
rootNodeID := strings.TrimSpace(item.RootNodeID)
|
||||
if rootNodeID == "" {
|
||||
continue
|
||||
}
|
||||
document, found, unexplored, err := firstValidateDocument(
|
||||
ctx, api, rootNodeID, settings, &listings,
|
||||
readable, unexplored, hitCap, err := sampleWorkspace(
|
||||
ctx, api, item.Name, rootNodeID, settings,
|
||||
&listings, &probes, &sawDocument, &lastErr,
|
||||
)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("workspace %q: %w", item.Name, err)
|
||||
|
|
@ -157,29 +160,24 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
|
|||
workspaceFail++
|
||||
continue
|
||||
}
|
||||
if readable {
|
||||
return nil
|
||||
}
|
||||
if unexplored {
|
||||
inconclusive = true
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
sawDocument = true
|
||||
probes++
|
||||
if err := verifyDocument(ctx, api, document); err != nil {
|
||||
lastErr = fmt.Errorf("workspace %q document %q: %w", item.Name, document.Name, err)
|
||||
if ctx.Err() != nil || isContextError(err) {
|
||||
return fmt.Errorf("validate DingTalk data source: %w", lastErr)
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
// Once the cap is spent no further probe can succeed, so walking on
|
||||
// could only find an unlistable folder and turn the failed probes
|
||||
// into acceptance.
|
||||
if probes >= maxValidateProbes {
|
||||
probeCapHit = hitCap || i+1 < len(workspaces)
|
||||
break
|
||||
}
|
||||
}
|
||||
// Folders were left unexplored, because the listing budget ran out or a
|
||||
// folder could not be listed, so an unreadable document elsewhere does not
|
||||
// prove the data source is unusable.
|
||||
// prove the data source is unusable. A probe cap hit in that situation
|
||||
// stays quiet too: the walk cannot claim it sampled every visible document.
|
||||
if inconclusive {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -194,6 +192,12 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
|
|||
if lastErr == nil {
|
||||
return nil
|
||||
}
|
||||
if probeCapHit {
|
||||
return fmt.Errorf(
|
||||
"validate DingTalk data source: %w; only the first %d documents were probed",
|
||||
lastErr, maxValidateProbes,
|
||||
)
|
||||
}
|
||||
return fmt.Errorf("validate DingTalk data source: %w", lastErr)
|
||||
}
|
||||
|
||||
|
|
@ -205,45 +209,83 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
|
|||
// client's retries of each).
|
||||
const maxValidateListings = 20
|
||||
|
||||
// firstValidateDocument walks a workspace breadth-first, the way scanWorkspace
|
||||
// does, and returns the first document it finds. It requests one page at a
|
||||
// time instead of listing whole folders, so the budget bounds real requests.
|
||||
// unexplored reports that some nodes were never seen, because the budget ran
|
||||
// out or a folder could not be listed, so the walk cannot prove the workspace
|
||||
// holds no readable document.
|
||||
// sampleWorkspace walks one workspace breadth-first, the way scanWorkspace
|
||||
// does, and probes ingestible documents until one is readable, the probe cap
|
||||
// is spent, or the walk runs out of nodes. It requests one page at a time
|
||||
// instead of listing whole folders, so the budget bounds real requests.
|
||||
//
|
||||
// unexplored reports that some folder was never opened or could not be listed,
|
||||
// so a failed probe elsewhere must not reject the data source. readable reports
|
||||
// that a probe succeeded. The walk stops as soon as the failed probe that
|
||||
// spends maxValidateProbes returns, without another listing; capHit then
|
||||
// reports that nodes of this workspace were still left to visit. The caller
|
||||
// turns that into an error only when the walk is conclusive.
|
||||
//
|
||||
// A folder's later pages are held back once the listings still in the budget
|
||||
// are no longer enough to open every sibling already queued, so each of those
|
||||
// siblings still gets one page. The unread tail is not itself "unexplored"
|
||||
// when a document was probed after it was skipped: the reservation is what
|
||||
// made room for that probe. A tail skipped with no probe after it stays
|
||||
// inconclusive, because the only readable document may sit on a page the
|
||||
// sample never requested. A folder that receives no page at all, a subfolder
|
||||
// listing error, a repeated page token, or a tail with no sibling waiting all
|
||||
// stay inconclusive too.
|
||||
//
|
||||
// Any failed page of the workspace root is a hard error, matching listNodes
|
||||
// during sync. Subfolder listing errors stay inconclusive unless the request
|
||||
// was cancelled or timed out.
|
||||
//
|
||||
// The walk selects nodes with the same settings-aware predicate the picker and
|
||||
// the sync use, so a type the data source does not ingest is never probed.
|
||||
func firstValidateDocument(
|
||||
func sampleWorkspace(
|
||||
ctx context.Context,
|
||||
api dingTalkAPI,
|
||||
workspaceName string,
|
||||
rootNodeID string,
|
||||
settings documentSettings,
|
||||
listings *int,
|
||||
) (document node, found, unexplored bool, err error) {
|
||||
probes *int,
|
||||
sawDocument *bool,
|
||||
lastErr *error,
|
||||
) (readable, unexplored, capHit bool, err error) {
|
||||
queue := []string{rootNodeID}
|
||||
visited := map[string]struct{}{rootNodeID: {}}
|
||||
uncoveredSkip := false
|
||||
for len(queue) > 0 {
|
||||
parent := queue[0]
|
||||
queue = queue[1:]
|
||||
// Finish every page of a folder before descending, like listNodes
|
||||
// does for scanWorkspace: a document on a later page of this folder
|
||||
// must be found before the budget is spent on its subfolders.
|
||||
// Siblings already on the queue stay ahead of those subfolders.
|
||||
var subfolders []string
|
||||
pageToken := ""
|
||||
seenTokens := map[string]struct{}{}
|
||||
for {
|
||||
// Reserve one listing per sibling already queued before spending
|
||||
// another page on this folder. The root's own pages have an empty
|
||||
// queue here — its children are not queued until the root is
|
||||
// finished — so a document on the root's next page is still read
|
||||
// before the walk descends.
|
||||
if pageToken != "" && *listings+len(queue) >= maxValidateListings {
|
||||
if len(queue) == 0 {
|
||||
unexplored = true
|
||||
} else {
|
||||
uncoveredSkip = true
|
||||
}
|
||||
break
|
||||
}
|
||||
rootPage := parent == rootNodeID && pageToken == ""
|
||||
if !rootPage {
|
||||
if *listings >= maxValidateListings {
|
||||
return node{}, false, true, nil
|
||||
return false, true, false, nil
|
||||
}
|
||||
*listings++
|
||||
}
|
||||
children, next, listErr := api.listNodesPage(ctx, parent, pageToken)
|
||||
if listErr != nil {
|
||||
if rootPage || ctx.Err() != nil || isContextError(listErr) {
|
||||
return node{}, false, false, listErr
|
||||
if parent == rootNodeID || ctx.Err() != nil || isContextError(listErr) {
|
||||
return false, false, false, listErr
|
||||
}
|
||||
// The documents this operator can read may live in exactly
|
||||
// this folder, so its failure leaves the walk inconclusive
|
||||
|
|
@ -251,9 +293,27 @@ func firstValidateDocument(
|
|||
unexplored = true
|
||||
break
|
||||
}
|
||||
for _, child := range children {
|
||||
for i, child := range children {
|
||||
if child.isIngestible(settings) {
|
||||
return child, true, unexplored, nil
|
||||
*probes++
|
||||
*sawDocument = true
|
||||
uncoveredSkip = false
|
||||
probeErr := verifyDocument(ctx, api, child)
|
||||
if probeErr != nil {
|
||||
if ctx.Err() != nil || isContextError(probeErr) {
|
||||
return false, false, false, fmt.Errorf("document %q: %w", child.Name, probeErr)
|
||||
}
|
||||
*lastErr = fmt.Errorf(
|
||||
"workspace %q document %q: %w", workspaceName, child.Name, probeErr,
|
||||
)
|
||||
if *probes >= maxValidateProbes {
|
||||
more := i+1 < len(children) || len(subfolders) > 0 ||
|
||||
next != "" || len(queue) > 0
|
||||
return false, unexplored, more, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
return true, false, false, nil
|
||||
}
|
||||
if child.isFolder() || child.HasChildren {
|
||||
if _, seen := visited[child.ID]; seen {
|
||||
|
|
@ -277,7 +337,10 @@ func firstValidateDocument(
|
|||
}
|
||||
queue = append(queue, subfolders...)
|
||||
}
|
||||
return node{}, false, unexplored, nil
|
||||
if uncoveredSkip {
|
||||
unexplored = true
|
||||
}
|
||||
return false, unexplored, false, nil
|
||||
}
|
||||
|
||||
// verifyDocument proves one visible document is readable. Native documents are
|
||||
|
|
|
|||
|
|
@ -147,6 +147,11 @@ func TestValidateReportsWhenNoVisibleDocumentIsReadable(t *testing.T) {
|
|||
t.Fatalf("Validate error should carry %s, got: %v", want, err)
|
||||
}
|
||||
}
|
||||
// Both documents were probed, so the cap did not stop the walk and the
|
||||
// error must not claim that it did.
|
||||
if strings.Contains(err.Error(), "only the first") {
|
||||
t.Fatalf("Validate error should not mention the probe cap, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A tenant with no documents anywhere leaves nothing to read, so the
|
||||
|
|
@ -194,9 +199,10 @@ func TestValidateSurvivesWorkspaceListingFailure(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// An app without the document read permission fails every probe. Validate must
|
||||
// probe at most one document per workspace and stop after maxValidateProbes,
|
||||
// instead of firing one call per document across the whole tenant.
|
||||
// An app without the document read permission fails every probe. Validate keeps
|
||||
// going to the next document in the same workspace, but it still stops after
|
||||
// maxValidateProbes reads and says so, instead of firing one call per document
|
||||
// across the whole tenant.
|
||||
func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
nodes: map[string][]node{},
|
||||
|
|
@ -216,20 +222,32 @@ func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {
|
|||
}
|
||||
|
||||
c := testConnector(api)
|
||||
if err := c.Validate(context.Background(), testConfig()); err == nil {
|
||||
err := c.Validate(context.Background(), testConfig())
|
||||
if err == nil {
|
||||
t.Fatal("Validate must fail when no visible document can be read")
|
||||
}
|
||||
total := 0
|
||||
for _, n := range api.blockCalls {
|
||||
total += n
|
||||
}
|
||||
if total > maxValidateProbes {
|
||||
t.Fatalf("Validate made %d document probes, want at most %d", total, maxValidateProbes)
|
||||
if total != maxValidateProbes {
|
||||
t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)
|
||||
}
|
||||
for w := 0; w < workspaces; w++ {
|
||||
if api.blockCalls[fmt.Sprintf("doc-%d-1", w)] != 0 {
|
||||
t.Fatalf("workspace root-%d was probed past its first document", w)
|
||||
}
|
||||
// The first workspace has four unreadable documents, so the cap must be
|
||||
// spent there and on the next workspace's first document — not one probe
|
||||
// per workspace, and not on a third workspace.
|
||||
if api.blockCalls["doc-0-1"] != 1 {
|
||||
t.Fatal("a failed probe must not stop the workspace from trying its next document")
|
||||
}
|
||||
if api.blockCalls["doc-1-0"] != 1 || api.blockCalls["doc-1-1"] != 0 || api.blockCalls["doc-2-0"] != 0 {
|
||||
t.Fatalf("probe cap was not applied across workspaces, calls=%v", api.blockCalls)
|
||||
}
|
||||
wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)
|
||||
if !strings.Contains(err.Error(), wantNote) {
|
||||
t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), `document "doc-1-0"`) {
|
||||
t.Fatalf("Validate error should name the last probed document, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -554,10 +572,14 @@ func TestValidateBudgetCountsNodeListingPages(t *testing.T) {
|
|||
|
||||
// A folder's later pages must be read before its subfolders: otherwise a root
|
||||
// whose first page is all folders spends the budget below them and never sees
|
||||
// the document on the root's second page.
|
||||
// the document on the root's second page. The document is unreadable and the
|
||||
// twenty subfolders cannot all be listed after that page spends a budget unit,
|
||||
// so the rest of the tree is unknown and Validate accepts.
|
||||
func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
probed := false
|
||||
folderLists := 0
|
||||
folderListsBeforeProbe := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
|
|
@ -578,6 +600,12 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
|
|||
_, _ = w.Write([]byte(
|
||||
`{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
default:
|
||||
mu.Lock()
|
||||
folderLists++
|
||||
if !probed {
|
||||
folderListsBeforeProbe++
|
||||
}
|
||||
mu.Unlock()
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))
|
||||
}
|
||||
case strings.HasPrefix(r.URL.Path, "/v1.0/doc/suites/documents/"):
|
||||
|
|
@ -598,8 +626,14 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
|
|||
if !probed {
|
||||
t.Fatalf("the document on the root's second page was never probed (err=%v)", err)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("Validate must report the unreadable document instead of accepting")
|
||||
if folderListsBeforeProbe != 0 {
|
||||
t.Fatalf("listed %d subfolders before probing the root's second page", folderListsBeforeProbe)
|
||||
}
|
||||
if folderLists == 0 {
|
||||
t.Fatal("Validate stopped at the unreadable document instead of continuing the walk")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Validate must accept once the listing budget leaves folders unexplored, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -640,3 +674,530 @@ func TestValidateStopsOnRepeatedPageToken(t *testing.T) {
|
|||
t.Fatalf("folder was listed %d times, want 2 (first page + the repeated token once)", folderRequests)
|
||||
}
|
||||
}
|
||||
|
||||
// The first document of a workspace can be unreadable while a later one in the
|
||||
// same folder is not. Stopping at the first probe would reject a data source
|
||||
// the operator can actually sync.
|
||||
func TestValidateTriesTheNextDocumentAfterAFailedProbe(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
|
||||
nodes: map[string][]node{
|
||||
"root-a": {
|
||||
{ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
{ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
},
|
||||
},
|
||||
blocks: map[string][]json.RawMessage{
|
||||
"doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},
|
||||
},
|
||||
blockErrors: map[string]error{
|
||||
"doc-bad": errors.New("forbidden.accessDenied: the operator has no permission"),
|
||||
},
|
||||
}
|
||||
|
||||
if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept once a later document in the folder is readable, got: %v", err)
|
||||
}
|
||||
if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {
|
||||
t.Fatalf("both documents should be probed, calls=%v", api.blockCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// A readable document that lives in a later sibling folder must still be
|
||||
// reached after the first folder's document fails. The same walk, when every
|
||||
// document fails, reports the later one: the workspace is conclusive.
|
||||
func TestValidateTriesADocumentInALaterSiblingFolder(t *testing.T) {
|
||||
denied := errors.New("forbidden.accessDenied: the operator has no permission")
|
||||
t.Run("later folder readable", func(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
|
||||
nodes: map[string][]node{
|
||||
"root-a": {
|
||||
{ID: "folder-a", Name: "A", Type: "FOLDER"},
|
||||
{ID: "folder-b", Name: "B", Type: "FOLDER"},
|
||||
},
|
||||
"folder-a": {
|
||||
{ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
},
|
||||
"folder-b": {
|
||||
{ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
},
|
||||
},
|
||||
blocks: map[string][]json.RawMessage{
|
||||
"doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},
|
||||
},
|
||||
blockErrors: map[string]error{"doc-bad": denied},
|
||||
}
|
||||
if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept the readable document in the later folder, got: %v", err)
|
||||
}
|
||||
if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {
|
||||
t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)
|
||||
}
|
||||
})
|
||||
t.Run("every document unreadable", func(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
|
||||
nodes: map[string][]node{
|
||||
"root-a": {
|
||||
{ID: "folder-a", Name: "A", Type: "FOLDER"},
|
||||
{ID: "folder-b", Name: "B", Type: "FOLDER"},
|
||||
},
|
||||
"folder-a": {
|
||||
{ID: "doc-first", Name: "First", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
},
|
||||
"folder-b": {
|
||||
{ID: "doc-second", Name: "Second", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
|
||||
},
|
||||
},
|
||||
blockErrors: map[string]error{"doc-first": denied, "doc-second": denied},
|
||||
}
|
||||
err := testConnector(api).Validate(context.Background(), testConfig())
|
||||
if err == nil || !strings.Contains(err.Error(), `document "Second"`) {
|
||||
t.Fatalf("Validate must report the later unreadable document, got: %v", err)
|
||||
}
|
||||
if api.blockCalls["doc-first"] != 1 || api.blockCalls["doc-second"] != 1 {
|
||||
t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// An unreadable document on a later root page is conclusive when the rest of
|
||||
// the workspace fits in the listing budget. The page is still read before any
|
||||
// subfolder, and the walk then finishes those subfolders instead of stopping
|
||||
// on the first failed probe.
|
||||
func TestValidateReportsUnreadableDocumentOnALaterRootPage(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
probed := false
|
||||
folderLists := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1.0/oauth2/accessToken":
|
||||
_, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
|
||||
case "/v2.0/wiki/workspaces":
|
||||
_, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
|
||||
case "/v2.0/wiki/nodes":
|
||||
parent := r.URL.Query().Get("parentNodeId")
|
||||
switch {
|
||||
case parent == "root-a" && r.URL.Query().Get("nextToken") == "":
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))
|
||||
case parent == "root-a":
|
||||
_, _ = w.Write([]byte(
|
||||
`{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
default:
|
||||
mu.Lock()
|
||||
folderLists++
|
||||
mu.Unlock()
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))
|
||||
}
|
||||
case "/v1.0/doc/suites/documents/doc/blocks":
|
||||
mu.Lock()
|
||||
probed = true
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !probed || folderLists != 1 {
|
||||
t.Fatalf("probed=%v folderLists=%d, want the page probed and its subfolder listed", probed, folderLists)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), `document "Doc"`) {
|
||||
t.Fatalf("Validate must report the unreadable document, got: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "only the first") {
|
||||
t.Fatalf("one probed document must not be described as the probe cap, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A root listing that fails on a later page is the same failure sync sees from
|
||||
// listNodes: the workspace is not "unexplored", even when an earlier page
|
||||
// already named a folder that holds a readable document. Another workspace can
|
||||
// still prove the credentials, as a failed first page already can.
|
||||
func TestValidateRejectsAFailedLaterRootPage(t *testing.T) {
|
||||
const workspaces = `{"workspaces":[` +
|
||||
`{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` +
|
||||
`{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`
|
||||
handler := func(extraWorkspace bool) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1.0/oauth2/accessToken":
|
||||
_, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
|
||||
case "/v2.0/wiki/workspaces":
|
||||
if extraWorkspace {
|
||||
_, _ = w.Write([]byte(workspaces))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
|
||||
case "/v2.0/wiki/nodes":
|
||||
parent := r.URL.Query().Get("parentNodeId")
|
||||
switch {
|
||||
case parent == "root-a" && r.URL.Query().Get("nextToken") == "":
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))
|
||||
case parent == "root-a":
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"code":"invalidRequest","message":"bad page"}`))
|
||||
case parent == "folder":
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"hidden","name":"Hidden",` +
|
||||
`"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Handbook",` +
|
||||
`"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
}
|
||||
case "/v1.0/doc/suites/documents/hidden/blocks", "/v1.0/doc/suites/documents/doc-b/blocks":
|
||||
_, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run("only workspace", func(t *testing.T) {
|
||||
server := httptest.NewServer(handler(false))
|
||||
defer server.Close()
|
||||
err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
|
||||
if err == nil || !strings.Contains(err.Error(), "status=400") ||
|
||||
!strings.Contains(err.Error(), `workspace "A"`) {
|
||||
t.Fatalf("Validate must surface the root page failure, got: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("later workspace readable", func(t *testing.T) {
|
||||
server := httptest.NewServer(handler(true))
|
||||
defer server.Close()
|
||||
if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must survive one workspace whose later root page fails, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The first folder under the root can hold more non-document pages than the
|
||||
// listing budget. Those pages must not be spent to the end while a sibling
|
||||
// folder is still queued: the sibling is opened, and a document there is
|
||||
// probed. A short folder is still finished first, so a document on its second
|
||||
// page is not skipped just because a sibling is waiting.
|
||||
func TestValidateOpensSiblingFolderBeforeSpendingTheListingBudget(t *testing.T) {
|
||||
const fatPages = 30
|
||||
type listingStats struct {
|
||||
fatPages int
|
||||
probed map[string]int
|
||||
}
|
||||
start := func(t *testing.T, siblingReadable, docOnSecondPage bool) (*httptest.Server, *listingStats) {
|
||||
t.Helper()
|
||||
stats := &listingStats{probed: map[string]int{}}
|
||||
var mu sync.Mutex
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1.0/oauth2/accessToken":
|
||||
_, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
|
||||
case "/v2.0/wiki/workspaces":
|
||||
_, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
|
||||
case "/v2.0/wiki/nodes":
|
||||
parent := r.URL.Query().Get("parentNodeId")
|
||||
token := r.URL.Query().Get("nextToken")
|
||||
switch parent {
|
||||
case "root-a":
|
||||
_, _ = w.Write([]byte(`{"nodes":[` +
|
||||
`{"nodeId":"folder-a","type":"FOLDER"},` +
|
||||
`{"nodeId":"folder-b","type":"FOLDER"}]}`))
|
||||
case "folder-a":
|
||||
mu.Lock()
|
||||
stats.fatPages++
|
||||
page := stats.fatPages
|
||||
mu.Unlock()
|
||||
if token != "" && token != strconv.Itoa(page-1) {
|
||||
t.Errorf("folder-a page token = %q, previous page was %d", token, page-1)
|
||||
}
|
||||
item := fmt.Sprintf(
|
||||
`{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}`, page,
|
||||
)
|
||||
if docOnSecondPage && page == 2 {
|
||||
item = `{"nodeId":"doc-early","name":"Early",` +
|
||||
`"type":"FILE","category":"ALIDOC","extension":"adoc"}`
|
||||
}
|
||||
next := ""
|
||||
if page < fatPages {
|
||||
next = strconv.Itoa(page)
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, `{"nodes":[%s],"nextToken":%q}`, item, next)
|
||||
case "folder-b":
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Sibling",` +
|
||||
`"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
default:
|
||||
const prefix = "/v1.0/doc/suites/documents/"
|
||||
if !strings.HasPrefix(r.URL.Path, prefix) || !strings.HasSuffix(r.URL.Path, "/blocks") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), "/blocks")
|
||||
mu.Lock()
|
||||
stats.probed[id]++
|
||||
mu.Unlock()
|
||||
if id == "doc-b" && !siblingReadable {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))
|
||||
}
|
||||
}))
|
||||
return server, stats
|
||||
}
|
||||
|
||||
t.Run("sibling document readable", func(t *testing.T) {
|
||||
server, stats := start(t, true, false)
|
||||
defer server.Close()
|
||||
if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept the sibling document, got: %v", err)
|
||||
}
|
||||
if stats.probed["doc-b"] != 1 {
|
||||
t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)
|
||||
}
|
||||
if stats.fatPages >= maxValidateListings {
|
||||
t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",
|
||||
stats.fatPages, maxValidateListings)
|
||||
}
|
||||
})
|
||||
t.Run("sibling document unreadable", func(t *testing.T) {
|
||||
server, stats := start(t, false, false)
|
||||
defer server.Close()
|
||||
err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
|
||||
if err == nil || !strings.Contains(err.Error(), `document "Sibling"`) {
|
||||
t.Fatalf("Validate must report the sibling document, got: %v", err)
|
||||
}
|
||||
if stats.probed["doc-b"] != 1 {
|
||||
t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)
|
||||
}
|
||||
if stats.fatPages >= maxValidateListings {
|
||||
t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",
|
||||
stats.fatPages, maxValidateListings)
|
||||
}
|
||||
})
|
||||
t.Run("short folder still yields its second page", func(t *testing.T) {
|
||||
server, stats := start(t, false, true)
|
||||
defer server.Close()
|
||||
if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept the document on the short folder's second page, got: %v", err)
|
||||
}
|
||||
if stats.probed["doc-early"] != 1 || stats.probed["doc-b"] != 0 {
|
||||
t.Fatalf("probes=%v, want only the document on page 2", stats.probed)
|
||||
}
|
||||
if stats.fatPages != 2 {
|
||||
t.Fatalf("folder-a was listed %d times, want 2", stats.fatPages)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Skipping the unread tail of a fat folder is what leaves room for its sibling.
|
||||
// When that sample contains no document, the tail might still hold the only
|
||||
// readable one, so an unreadable document in another workspace must not fail
|
||||
// the data source.
|
||||
func TestValidateSkippedTailWithoutAProbeStaysInconclusive(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
probed := false
|
||||
fatPages := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1.0/oauth2/accessToken":
|
||||
_, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
|
||||
case "/v2.0/wiki/workspaces":
|
||||
_, _ = w.Write([]byte(`{"workspaces":[` +
|
||||
`{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` +
|
||||
`{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`))
|
||||
case "/v2.0/wiki/nodes":
|
||||
parent := r.URL.Query().Get("parentNodeId")
|
||||
switch parent {
|
||||
case "root-a":
|
||||
_, _ = w.Write([]byte(`{"nodes":[` +
|
||||
`{"nodeId":"folder-a","type":"FOLDER"},` +
|
||||
`{"nodeId":"folder-b","type":"FOLDER"}]}`))
|
||||
case "folder-a":
|
||||
mu.Lock()
|
||||
fatPages++
|
||||
page := fatPages
|
||||
mu.Unlock()
|
||||
next := ""
|
||||
if page < 30 {
|
||||
next = strconv.Itoa(page)
|
||||
}
|
||||
_, _ = fmt.Fprintf(w,
|
||||
`{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,
|
||||
page, next)
|
||||
case "folder-b":
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf-b","type":"FILE",` +
|
||||
`"category":"FILE","extension":"pdf"}]}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Secret",` +
|
||||
`"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
|
||||
}
|
||||
case "/v1.0/doc/suites/documents/doc-b/blocks":
|
||||
mu.Lock()
|
||||
probed = true
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("Validate must accept when the skipped tail was never probed, got: %v", err)
|
||||
}
|
||||
if !probed {
|
||||
t.Fatal("the other workspace's document was never probed")
|
||||
}
|
||||
if fatPages >= maxValidateListings {
|
||||
t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)
|
||||
}
|
||||
}
|
||||
|
||||
// An unlistable folder already makes the walk inconclusive. Hitting the probe
|
||||
// cap afterwards must not turn that into a failure: a readable document may
|
||||
// still sit in the folder that could not be listed.
|
||||
func TestValidateProbeCapDoesNotOverrideAnInconclusiveWalk(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
nodes: map[string][]node{
|
||||
"root-deep": {{ID: "folder", Name: "Folder", Type: "FOLDER"}},
|
||||
},
|
||||
nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},
|
||||
blockErrors: map[string]error{},
|
||||
}
|
||||
api.workspaces = []workspace{{ID: "deep", RootNodeID: "root-deep", Name: "Deep"}}
|
||||
denied := errors.New("forbidden.accessDenied: the operator has no permission")
|
||||
for w := 0; w < maxValidateProbes+2; w++ {
|
||||
root := fmt.Sprintf("root-%d", w)
|
||||
id := fmt.Sprintf("doc-%d", w)
|
||||
api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})
|
||||
api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}
|
||||
api.blockErrors[id] = denied
|
||||
}
|
||||
|
||||
if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept an inconclusive walk even after the probe cap, got: %v", err)
|
||||
}
|
||||
total := 0
|
||||
for _, n := range api.blockCalls {
|
||||
total += n
|
||||
}
|
||||
if total != maxValidateProbes {
|
||||
t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)
|
||||
}
|
||||
}
|
||||
|
||||
// Once the probe cap is spent no later probe can succeed. A workspace walked
|
||||
// after that point must not get the chance to find an unlistable folder and
|
||||
// turn every failed probe into acceptance.
|
||||
func TestValidateStopsWalkingOnceTheProbeCapIsSpent(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
nodes: map[string][]node{},
|
||||
nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},
|
||||
blockErrors: map[string]error{},
|
||||
}
|
||||
denied := errors.New("forbidden.accessDenied: the operator has no permission")
|
||||
for w := 0; w < maxValidateProbes; w++ {
|
||||
root := fmt.Sprintf("root-%d", w)
|
||||
id := fmt.Sprintf("doc-%d", w)
|
||||
api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})
|
||||
api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}
|
||||
api.blockErrors[id] = denied
|
||||
}
|
||||
api.workspaces = append(api.workspaces, workspace{ID: "late", RootNodeID: "root-late", Name: "Late"})
|
||||
api.nodes["root-late"] = []node{{ID: "folder", Name: "Folder", Type: "FOLDER"}}
|
||||
|
||||
err := testConnector(api).Validate(context.Background(), testConfig())
|
||||
if err == nil || !strings.Contains(err.Error(), "no permission") {
|
||||
t.Fatalf("Validate must report the failed probes, got: %v", err)
|
||||
}
|
||||
wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)
|
||||
if !strings.Contains(err.Error(), wantNote) {
|
||||
t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A tail skipped to make room for siblings is only covered by a probe made
|
||||
// after it. A document probed before the skip says nothing about the skipped
|
||||
// pages, so an empty sibling must leave the walk inconclusive.
|
||||
func TestValidateSkippedTailIsNotCoveredByAnEarlierProbe(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
fatPages := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1.0/oauth2/accessToken":
|
||||
_, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
|
||||
case "/v2.0/wiki/workspaces":
|
||||
_, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
|
||||
case "/v2.0/wiki/nodes":
|
||||
switch r.URL.Query().Get("parentNodeId") {
|
||||
case "root-a":
|
||||
_, _ = w.Write([]byte(`{"nodes":[` +
|
||||
`{"nodeId":"doc-x","name":"X","type":"FILE","category":"ALIDOC","extension":"adoc"},` +
|
||||
`{"nodeId":"folder-a","type":"FOLDER"},` +
|
||||
`{"nodeId":"folder-b","type":"FOLDER"}]}`))
|
||||
case "folder-a":
|
||||
mu.Lock()
|
||||
fatPages++
|
||||
page := fatPages
|
||||
mu.Unlock()
|
||||
next := ""
|
||||
if page < 30 {
|
||||
next = strconv.Itoa(page)
|
||||
}
|
||||
_, _ = fmt.Fprintf(w,
|
||||
`{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,
|
||||
page, next)
|
||||
case "folder-b":
|
||||
_, _ = w.Write([]byte(`{"nodes":[]}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
case "/v1.0/doc/suites/documents/doc-x/blocks":
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
|
||||
t.Fatalf("Validate must accept when the skipped tail was never covered by a probe, got: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if fatPages >= maxValidateListings {
|
||||
t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)
|
||||
}
|
||||
}
|
||||
|
||||
// A probe cancelled by the caller's deadline still names the document, so the
|
||||
// operator can tell which read timed out.
|
||||
func TestValidateContextErrorNamesTheProbedDocument(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
|
||||
nodes: map[string][]node{
|
||||
"root-a": {{ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}},
|
||||
},
|
||||
blockErrors: map[string]error{"doc": context.DeadlineExceeded},
|
||||
}
|
||||
err := testConnector(api).Validate(context.Background(), testConfig())
|
||||
if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), `document "Handbook"`) {
|
||||
t.Fatalf("Validate must surface the timed-out probe with its document, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue