fix(confluence): stop Server listings that link past the last page

Some Server/Data Center builds keep returning _links.next from
/rest/api/space/{key}/content/page after the last page, with empty
results (#3596). The connector followed it until the 10000-hop cap, so a
sync ran for hours, imported nothing and then failed.

Stop after three consecutive empty pages that still link onward. A
single empty page is not taken as the end, since permission filtering
can empty one window mid-listing. A listing cut short this way is
reported as incomplete: its pages are imported, pages it did not reach
stay in the cursor, and deletion reconciliation is skipped for that run
so an early stop cannot mirror-delete documents.
main
wizardchen 2026-09-24 11:53:35 +08:00 committed by lyingbug
parent a1b0d0cb6c
commit eb888d0d3d
4 changed files with 158 additions and 14 deletions

View File

@ -21,6 +21,11 @@ const (
maxErrorResponseRunes = 1000
maxRetryDelay = 60 * time.Second
maxPaginationHops = 10000
// maxEmptyServerPages bounds how far a Server/Data Center page listing
// follows _links.next through empty pages. Some builds keep returning
// next past the end of a space (#3596); permission filtering can also
// empty one page mid-listing, so a single empty page is not the end.
maxEmptyServerPages = 3
)
type apiError struct {
@ -299,7 +304,11 @@ func withCloudPageQuery(next string) string {
return parsed.String()
}
func (c *client) pages(ctx context.Context, s space) ([]page, error) {
// pages lists every current page in a space. complete is false when a Server
// listing was cut short after maxEmptyServerPages empty pages that still
// linked onward; the caller must not treat pages missing from such a listing
// as deleted.
func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {
if c.cfg.cloud() {
var all []page
start := "/api/v2/spaces/" + url.PathEscape(s.ID) + "/pages?status=current&depth=all&limit=250"
@ -323,27 +332,39 @@ func (c *client) pages(ctx context.Context, s space) ([]page, error) {
}
return next, nil
})
return all, err
return all, true, err
}
var all []page
err := c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
complete = true
emptyPages := 0
err = c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
var result serverSpacePageList
if err := c.get(ctx, pageURL, &result); err != nil {
return "", err
}
for _, listed := range result.pages() {
listedPages := result.pages()
for _, listed := range listedPages {
if listed.Space.Key == "" {
listed.Space.Key, listed.Space.Name = s.Key, s.Name
}
all = append(all, listed)
}
next := result.nextLink()
if next != "" {
next = withServerPageExpand(next)
if next == "" {
return "", nil
}
return next, nil
if len(listedPages) > 0 {
emptyPages = 0
} else {
emptyPages++
if emptyPages >= maxEmptyServerPages {
complete = false
return "", nil
}
}
return withServerPageExpand(next), nil
})
return all, err
return all, complete, err
}
func (c *client) body(ctx context.Context, id string) (pageBody, error) {

View File

@ -3,6 +3,7 @@ package confluence
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
@ -31,7 +32,7 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
@ -49,6 +50,63 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
}
}
func TestServerPagesStopAfterRepeatedEmptyPages(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
w.Header().Set("Content-Type", "application/json")
results := `[]`
if r.URL.Query().Get("start") == "" {
results = `[{"id": "1", "title": "Root", "version": {"number": 1}}]`
}
next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", calls*100)
_, _ = fmt.Fprintf(w, `{"results": %s, "_links": {"next": %q}}`, results, next)
}))
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
if complete {
t.Fatal("a listing cut short by empty pages must not report complete")
}
if len(pages) != 1 || pages[0].ID != "1" {
t.Fatalf("pages = %#v", pages)
}
if calls != 1+maxEmptyServerPages {
t.Fatalf("calls = %d, want %d", calls, 1+maxEmptyServerPages)
}
}
func TestServerPagesContinuePastOneEmptyPage(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Query().Get("start") {
case "":
_, _ = w.Write([]byte(`{"results": [{"id": "1", "title": "A"}],
"_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}}`))
case "100":
// Every page in this window was filtered out by permissions.
_, _ = w.Write([]byte(`{"results": [],
"_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=200"}}`))
default:
_, _ = w.Write([]byte(`{"results": [{"id": "2", "title": "B"}]}`))
}
}))
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
if !complete || len(pages) != 2 {
t.Fatalf("complete = %v, pages = %#v", complete, pages)
}
}
func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -61,7 +119,7 @@ func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
pages, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
pages, _, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
@ -108,7 +166,7 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}
pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}

View File

@ -10,6 +10,7 @@ import (
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@ -174,10 +175,16 @@ func (c *Connector) fetchStream(
resourceID,
)
}
pages, err := client.pages(ctx, s)
pages, complete, err := client.pages(ctx, s)
if err != nil {
return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)
}
if !complete {
logger.Warnf(ctx,
"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
"imported %d pages and skipped deletion reconciliation",
s.Key, maxEmptyServerPages, len(pages))
}
priorPages, hadBaseline := baseline.SpacePages[resourceID]
if next.SpacePages[resourceID] == nil {
next.SpacePages[resourceID] = map[string]string{}
@ -221,7 +228,16 @@ func (c *Connector) fetchStream(
return nil, err
}
}
if hadBaseline {
if !complete {
// A cut-short listing proves nothing about the pages it did not
// reach: keep tracking them so a later complete listing can still
// reconcile them, and delete nothing now.
for id, version := range priorPages {
if _, exists := seen[id]; !exists {
next.SpacePages[resourceID][id] = version
}
}
} else if hadBaseline {
if len(pages) == 0 && len(priorPages) > 0 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",

View File

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
@ -30,6 +31,9 @@ type streamAPI struct {
bodyCalls int
bodyStatus map[string]int
listCalls int
// endlessNext mimics Server/Data Center builds that keep linking to a
// next page after the last one, returning empty results forever (#3596).
endlessNext bool
}
func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {
@ -66,8 +70,19 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
})
case path == "/wiki/rest/api/space/ENG/content/page":
a.listCalls++
if !a.endlessNext {
return a.jsonResponse(map[string]any{
"results": a.pageMaps(),
})
}
results := a.pageMaps()
if a.listCalls > 1 {
results = []any{}
}
next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)
return a.jsonResponse(map[string]any{
"results": a.pageMaps(),
"results": results,
"_links": map[string]any{"next": next},
})
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
if req.URL.Query().Get("expand") != "body.view,version,space" {
@ -387,3 +402,37 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
t.Fatalf("ListResources() with hostile webui = %#v, %v", resources, err)
}
}
func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {
for _, full := range []bool{false, true} {
t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}
h := &captureHandler{}
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
connector := newStreamConnector(api)
fetch := connector.FetchStream
if full {
fetch = connector.FetchFullStream
}
got, err := fetch(context.Background(), streamConfig(), old, h)
if err != nil {
t.Fatalf("fetch: %v", err)
}
if api.listCalls != 1+maxEmptyServerPages {
t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
}
for _, item := range h.items {
if item.IsDeleted {
t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)
}
}
if len(h.items) != 1 || h.items[0].ExternalID != "p1" {
t.Fatalf("items = %#v, want the changed page p1 only", h.items)
}
pages := decodeCursor(got).SpacePages["1"]
if pages["p1"] != "v:2" || pages["p2"] != "v:1" {
t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)
}
})
}
}