fix(confluence): stop Server listings that link past the last page
Some Server/Data Center builds keep returning _links.next from
/rest/api/space/{key}/content/page after the last page, with empty
results (#3596). The connector followed it until the 10000-hop cap, so a
sync ran for hours, imported nothing and then failed.
Stop after three consecutive empty pages that still link onward. A
single empty page is not taken as the end, since permission filtering
can empty one window mid-listing. A listing cut short this way is
reported as incomplete: its pages are imported, pages it did not reach
stay in the cursor, and deletion reconciliation is skipped for that run
so an early stop cannot mirror-delete documents.
main
parent
a1b0d0cb6c
commit
eb888d0d3d
|
|
@ -21,6 +21,11 @@ const (
|
|||
maxErrorResponseRunes = 1000
|
||||
maxRetryDelay = 60 * time.Second
|
||||
maxPaginationHops = 10000
|
||||
// maxEmptyServerPages bounds how far a Server/Data Center page listing
|
||||
// follows _links.next through empty pages. Some builds keep returning
|
||||
// next past the end of a space (#3596); permission filtering can also
|
||||
// empty one page mid-listing, so a single empty page is not the end.
|
||||
maxEmptyServerPages = 3
|
||||
)
|
||||
|
||||
type apiError struct {
|
||||
|
|
@ -299,7 +304,11 @@ func withCloudPageQuery(next string) string {
|
|||
return parsed.String()
|
||||
}
|
||||
|
||||
func (c *client) pages(ctx context.Context, s space) ([]page, error) {
|
||||
// pages lists every current page in a space. complete is false when a Server
|
||||
// listing was cut short after maxEmptyServerPages empty pages that still
|
||||
// linked onward; the caller must not treat pages missing from such a listing
|
||||
// as deleted.
|
||||
func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {
|
||||
if c.cfg.cloud() {
|
||||
var all []page
|
||||
start := "/api/v2/spaces/" + url.PathEscape(s.ID) + "/pages?status=current&depth=all&limit=250"
|
||||
|
|
@ -323,27 +332,39 @@ func (c *client) pages(ctx context.Context, s space) ([]page, error) {
|
|||
}
|
||||
return next, nil
|
||||
})
|
||||
return all, err
|
||||
return all, true, err
|
||||
}
|
||||
var all []page
|
||||
err := c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
|
||||
complete = true
|
||||
emptyPages := 0
|
||||
err = c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
|
||||
var result serverSpacePageList
|
||||
if err := c.get(ctx, pageURL, &result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, listed := range result.pages() {
|
||||
listedPages := result.pages()
|
||||
for _, listed := range listedPages {
|
||||
if listed.Space.Key == "" {
|
||||
listed.Space.Key, listed.Space.Name = s.Key, s.Name
|
||||
}
|
||||
all = append(all, listed)
|
||||
}
|
||||
next := result.nextLink()
|
||||
if next != "" {
|
||||
next = withServerPageExpand(next)
|
||||
if next == "" {
|
||||
return "", nil
|
||||
}
|
||||
return next, nil
|
||||
if len(listedPages) > 0 {
|
||||
emptyPages = 0
|
||||
} else {
|
||||
emptyPages++
|
||||
if emptyPages >= maxEmptyServerPages {
|
||||
complete = false
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
return withServerPageExpand(next), nil
|
||||
})
|
||||
return all, err
|
||||
return all, complete, err
|
||||
}
|
||||
|
||||
func (c *client) body(ctx context.Context, id string) (pageBody, error) {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package confluence
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
|
@ -31,7 +32,7 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
|
|||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
|
||||
pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
|
||||
pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -49,6 +50,63 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestServerPagesStopAfterRepeatedEmptyPages(t *testing.T) {
|
||||
calls := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
results := `[]`
|
||||
if r.URL.Query().Get("start") == "" {
|
||||
results = `[{"id": "1", "title": "Root", "version": {"number": 1}}]`
|
||||
}
|
||||
next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", calls*100)
|
||||
_, _ = fmt.Fprintf(w, `{"results": %s, "_links": {"next": %q}}`, results, next)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
|
||||
pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if complete {
|
||||
t.Fatal("a listing cut short by empty pages must not report complete")
|
||||
}
|
||||
if len(pages) != 1 || pages[0].ID != "1" {
|
||||
t.Fatalf("pages = %#v", pages)
|
||||
}
|
||||
if calls != 1+maxEmptyServerPages {
|
||||
t.Fatalf("calls = %d, want %d", calls, 1+maxEmptyServerPages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerPagesContinuePastOneEmptyPage(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Query().Get("start") {
|
||||
case "":
|
||||
_, _ = w.Write([]byte(`{"results": [{"id": "1", "title": "A"}],
|
||||
"_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}}`))
|
||||
case "100":
|
||||
// Every page in this window was filtered out by permissions.
|
||||
_, _ = w.Write([]byte(`{"results": [],
|
||||
"_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=200"}}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"results": [{"id": "2", "title": "B"}]}`))
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
|
||||
pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !complete || len(pages) != 2 {
|
||||
t.Fatalf("complete = %v, pages = %#v", complete, pages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
|
@ -61,7 +119,7 @@ func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
|
|||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
|
||||
pages, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
|
||||
pages, _, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -108,7 +166,7 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
|
|||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}
|
||||
pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
|
||||
pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
|
|
@ -174,10 +175,16 @@ func (c *Connector) fetchStream(
|
|||
resourceID,
|
||||
)
|
||||
}
|
||||
pages, err := client.pages(ctx, s)
|
||||
pages, complete, err := client.pages(ctx, s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)
|
||||
}
|
||||
if !complete {
|
||||
logger.Warnf(ctx,
|
||||
"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
|
||||
"imported %d pages and skipped deletion reconciliation",
|
||||
s.Key, maxEmptyServerPages, len(pages))
|
||||
}
|
||||
priorPages, hadBaseline := baseline.SpacePages[resourceID]
|
||||
if next.SpacePages[resourceID] == nil {
|
||||
next.SpacePages[resourceID] = map[string]string{}
|
||||
|
|
@ -221,7 +228,16 @@ func (c *Connector) fetchStream(
|
|||
return nil, err
|
||||
}
|
||||
}
|
||||
if hadBaseline {
|
||||
if !complete {
|
||||
// A cut-short listing proves nothing about the pages it did not
|
||||
// reach: keep tracking them so a later complete listing can still
|
||||
// reconcile them, and delete nothing now.
|
||||
for id, version := range priorPages {
|
||||
if _, exists := seen[id]; !exists {
|
||||
next.SpacePages[resourceID][id] = version
|
||||
}
|
||||
}
|
||||
} else if hadBaseline {
|
||||
if len(pages) == 0 && len(priorPages) > 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
|
@ -30,6 +31,9 @@ type streamAPI struct {
|
|||
bodyCalls int
|
||||
bodyStatus map[string]int
|
||||
listCalls int
|
||||
// endlessNext mimics Server/Data Center builds that keep linking to a
|
||||
// next page after the last one, returning empty results forever (#3596).
|
||||
endlessNext bool
|
||||
}
|
||||
|
||||
func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {
|
||||
|
|
@ -66,8 +70,19 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
|
|||
})
|
||||
case path == "/wiki/rest/api/space/ENG/content/page":
|
||||
a.listCalls++
|
||||
if !a.endlessNext {
|
||||
return a.jsonResponse(map[string]any{
|
||||
"results": a.pageMaps(),
|
||||
})
|
||||
}
|
||||
results := a.pageMaps()
|
||||
if a.listCalls > 1 {
|
||||
results = []any{}
|
||||
}
|
||||
next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)
|
||||
return a.jsonResponse(map[string]any{
|
||||
"results": a.pageMaps(),
|
||||
"results": results,
|
||||
"_links": map[string]any{"next": next},
|
||||
})
|
||||
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
|
||||
if req.URL.Query().Get("expand") != "body.view,version,space" {
|
||||
|
|
@ -387,3 +402,37 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
|
|||
t.Fatalf("ListResources() with hostile webui = %#v, %v", resources, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {
|
||||
for _, full := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
|
||||
api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}
|
||||
h := &captureHandler{}
|
||||
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
|
||||
connector := newStreamConnector(api)
|
||||
fetch := connector.FetchStream
|
||||
if full {
|
||||
fetch = connector.FetchFullStream
|
||||
}
|
||||
got, err := fetch(context.Background(), streamConfig(), old, h)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if api.listCalls != 1+maxEmptyServerPages {
|
||||
t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
|
||||
}
|
||||
for _, item := range h.items {
|
||||
if item.IsDeleted {
|
||||
t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)
|
||||
}
|
||||
}
|
||||
if len(h.items) != 1 || h.items[0].ExternalID != "p1" {
|
||||
t.Fatalf("items = %#v, want the changed page p1 only", h.items)
|
||||
}
|
||||
pages := decodeCursor(got).SpacePages["1"]
|
||||
if pages["p1"] != "v:2" || pages["p2"] != "v:1" {
|
||||
t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue