diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go index dec73577..05c9d977 100644 --- a/internal/datasource/connector/confluence/client.go +++ b/internal/datasource/connector/confluence/client.go @@ -21,6 +21,11 @@ const ( maxErrorResponseRunes = 1000 maxRetryDelay = 60 * time.Second maxPaginationHops = 10000 + // maxEmptyServerPages bounds how far a Server/Data Center page listing + // follows _links.next through empty pages. Some builds keep returning + // next past the end of a space (#3596); permission filtering can also + // empty one page mid-listing, so a single empty page is not the end. + maxEmptyServerPages = 3 ) type apiError struct { @@ -299,7 +304,11 @@ func withCloudPageQuery(next string) string { return parsed.String() } -func (c *client) pages(ctx context.Context, s space) ([]page, error) { +// pages lists every current page in a space. complete is false when a Server +// listing was cut short after maxEmptyServerPages empty pages that still +// linked onward; the caller must not treat pages missing from such a listing +// as deleted. +func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) { if c.cfg.cloud() { var all []page start := "/api/v2/spaces/" + url.PathEscape(s.ID) + "/pages?status=current&depth=all&limit=250" @@ -323,27 +332,39 @@ func (c *client) pages(ctx context.Context, s space) ([]page, error) { } return next, nil }) - return all, err + return all, true, err } var all []page - err := c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) { + complete = true + emptyPages := 0 + err = c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) { var result serverSpacePageList if err := c.get(ctx, pageURL, &result); err != nil { return "", err } - for _, listed := range result.pages() { + listedPages := result.pages() + for _, listed := range listedPages { if listed.Space.Key == "" { listed.Space.Key, listed.Space.Name = s.Key, s.Name } all = append(all, listed) } next := result.nextLink() - if next != "" { - next = withServerPageExpand(next) + if next == "" { + return "", nil } - return next, nil + if len(listedPages) > 0 { + emptyPages = 0 + } else { + emptyPages++ + if emptyPages >= maxEmptyServerPages { + complete = false + return "", nil + } + } + return withServerPageExpand(next), nil }) - return all, err + return all, complete, err } func (c *client) body(ctx context.Context, id string) (pageBody, error) { diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go index 9f04b729..a0407e87 100644 --- a/internal/datasource/connector/confluence/client_pages_test.go +++ b/internal/datasource/connector/confluence/client_pages_test.go @@ -3,6 +3,7 @@ package confluence import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "testing" @@ -31,7 +32,7 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) { t.Cleanup(server.Close) c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} - pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) if err != nil { t.Fatal(err) } @@ -49,6 +50,63 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) { } } +func TestServerPagesStopAfterRepeatedEmptyPages(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + results := `[]` + if r.URL.Query().Get("start") == "" { + results = `[{"id": "1", "title": "Root", "version": {"number": 1}}]` + } + next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", calls*100) + _, _ = fmt.Fprintf(w, `{"results": %s, "_links": {"next": %q}}`, results, next) + })) + t.Cleanup(server.Close) + + c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} + pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"}) + if err != nil { + t.Fatal(err) + } + if complete { + t.Fatal("a listing cut short by empty pages must not report complete") + } + if len(pages) != 1 || pages[0].ID != "1" { + t.Fatalf("pages = %#v", pages) + } + if calls != 1+maxEmptyServerPages { + t.Fatalf("calls = %d, want %d", calls, 1+maxEmptyServerPages) + } +} + +func TestServerPagesContinuePastOneEmptyPage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Query().Get("start") { + case "": + _, _ = w.Write([]byte(`{"results": [{"id": "1", "title": "A"}], + "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}}`)) + case "100": + // Every page in this window was filtered out by permissions. + _, _ = w.Write([]byte(`{"results": [], + "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=200"}}`)) + default: + _, _ = w.Write([]byte(`{"results": [{"id": "2", "title": "B"}]}`)) + } + })) + t.Cleanup(server.Close) + + c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} + pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"}) + if err != nil { + t.Fatal(err) + } + if !complete || len(pages) != 2 { + t.Fatalf("complete = %v, pages = %#v", complete, pages) + } +} + func TestPagesAcceptsNestedContentEnvelope(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -61,7 +119,7 @@ func TestPagesAcceptsNestedContentEnvelope(t *testing.T) { t.Cleanup(server.Close) c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} - pages, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"}) + pages, _, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"}) if err != nil { t.Fatal(err) } @@ -108,7 +166,7 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) { t.Cleanup(server.Close) c := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()} - pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) if err != nil { t.Fatal(err) } diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go index 7b10b9df..b753484a 100644 --- a/internal/datasource/connector/confluence/connector.go +++ b/internal/datasource/connector/confluence/connector.go @@ -10,6 +10,7 @@ import ( htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -174,10 +175,16 @@ func (c *Connector) fetchStream( resourceID, ) } - pages, err := client.pages(ctx, s) + pages, complete, err := client.pages(ctx, s) if err != nil { return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err) } + if !complete { + logger.Warnf(ctx, + "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+ + "imported %d pages and skipped deletion reconciliation", + s.Key, maxEmptyServerPages, len(pages)) + } priorPages, hadBaseline := baseline.SpacePages[resourceID] if next.SpacePages[resourceID] == nil { next.SpacePages[resourceID] = map[string]string{} @@ -221,7 +228,16 @@ func (c *Connector) fetchStream( return nil, err } } - if hadBaseline { + if !complete { + // A cut-short listing proves nothing about the pages it did not + // reach: keep tracking them so a later complete listing can still + // reconcile them, and delete nothing now. + for id, version := range priorPages { + if _, exists := seen[id]; !exists { + next.SpacePages[resourceID][id] = version + } + } + } else if hadBaseline { if len(pages) == 0 && len(priorPages) > 0 { return nil, fmt.Errorf( "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline", diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go index b86ef979..07a024b0 100644 --- a/internal/datasource/connector/confluence/connector_stream_test.go +++ b/internal/datasource/connector/confluence/connector_stream_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "strings" @@ -30,6 +31,9 @@ type streamAPI struct { bodyCalls int bodyStatus map[string]int listCalls int + // endlessNext mimics Server/Data Center builds that keep linking to a + // next page after the last one, returning empty results forever (#3596). + endlessNext bool } func (a *streamAPI) jsonResponse(value any) (*http.Response, error) { @@ -66,8 +70,19 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { }) case path == "/wiki/rest/api/space/ENG/content/page": a.listCalls++ + if !a.endlessNext { + return a.jsonResponse(map[string]any{ + "results": a.pageMaps(), + }) + } + results := a.pageMaps() + if a.listCalls > 1 { + results = []any{} + } + next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100) return a.jsonResponse(map[string]any{ - "results": a.pageMaps(), + "results": results, + "_links": map[string]any{"next": next}, }) case strings.HasPrefix(path, "/wiki/rest/api/content/"): if req.URL.Query().Get("expand") != "body.view,version,space" { @@ -387,3 +402,37 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) { t.Fatalf("ListResources() with hostile webui = %#v, %v", resources, err) } } + +func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) { + for _, full := range []bool{false, true} { + t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) { + api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}} + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + connector := newStreamConnector(api) + fetch := connector.FetchStream + if full { + fetch = connector.FetchFullStream + } + got, err := fetch(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if api.listCalls != 1+maxEmptyServerPages { + t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages) + } + for _, item := range h.items { + if item.IsDeleted { + t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID) + } + } + if len(h.items) != 1 || h.items[0].ExternalID != "p1" { + t.Fatalf("items = %#v, want the changed page p1 only", h.items) + } + pages := decodeCursor(got).SpacePages["1"] + if pages["p1"] != "v:2" || pages["p2"] != "v:1" { + t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages) + } + }) + } +}