fix(yuque): 两处 offset 分页补跳数上限、重复页检测与 meta.total 提前结束 (#4415)
parent
073b5ea616
commit
4d767d9dc4
|
|
@ -23,6 +23,15 @@ const (
|
|||
// maxJSONResponseBytes bounds every API response body. Listings and document
|
||||
// payloads are small; a larger body means a broken or hostile server.
|
||||
maxJSONResponseBytes = 16 << 20
|
||||
|
||||
// maxPaginationHops bounds every offset-paginated walk in this file. The
|
||||
// only natural exit is a short page, which a gateway, CDN, or self-hosted
|
||||
// deployment that answers every offset with a full page never produces;
|
||||
// /users/{login}/repos carries no meta.total to stop on either. Without the
|
||||
// cap such a walk keeps calling the API until the sync task hits its
|
||||
// deadline, with `all` growing all the way. Mirrors the guard Notion,
|
||||
// Confluence, GitLab, and IMA already carry.
|
||||
maxPaginationHops = 10000
|
||||
)
|
||||
|
||||
// client wraps the Yuque Open API.
|
||||
|
|
@ -273,7 +282,12 @@ func (c *client) ListGroupRepos(ctx context.Context, login string) ([]v2Repo, er
|
|||
func (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2Repo, error) {
|
||||
var all []v2Repo
|
||||
offset := 0
|
||||
for {
|
||||
lastFirstID := int64(0)
|
||||
for hop := 0; ; hop++ {
|
||||
if hop >= maxPaginationHops {
|
||||
logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)
|
||||
return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)
|
||||
}
|
||||
q := buildQuery(map[string]string{
|
||||
"type": "Book",
|
||||
"offset": fmt.Sprintf("%d", offset),
|
||||
|
|
@ -287,6 +301,15 @@ func (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2R
|
|||
if len(resp.Data) < defaultPageSize {
|
||||
break
|
||||
}
|
||||
// A full page whose first id repeats the previous page means the server
|
||||
// is answering the same page for a growing offset; following it would
|
||||
// append the same repos forever.
|
||||
firstID := resp.Data[0].ID
|
||||
if firstID != 0 && firstID == lastFirstID {
|
||||
logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)
|
||||
return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)
|
||||
}
|
||||
lastFirstID = firstID
|
||||
offset += defaultPageSize
|
||||
}
|
||||
return all, nil
|
||||
|
|
@ -298,7 +321,12 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, error
|
|||
basePath := fmt.Sprintf("/api/v2/repos/%d/docs", bookID)
|
||||
var all []v2Doc
|
||||
offset := 0
|
||||
for {
|
||||
lastFirstID := int64(0)
|
||||
for hop := 0; ; hop++ {
|
||||
if hop >= maxPaginationHops {
|
||||
logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)
|
||||
return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)
|
||||
}
|
||||
q := buildQuery(map[string]string{
|
||||
"offset": fmt.Sprintf("%d", offset),
|
||||
"limit": fmt.Sprintf("%d", defaultPageSize),
|
||||
|
|
@ -311,6 +339,19 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, error
|
|||
if len(resp.Data) < defaultPageSize {
|
||||
break
|
||||
}
|
||||
// meta.total counts the whole listing, so a page that reaches it is the
|
||||
// last one; asking for the next offset would only fetch an empty page.
|
||||
if resp.Meta.Total > 0 && offset+len(resp.Data) >= resp.Meta.Total {
|
||||
break
|
||||
}
|
||||
// Same repeated-page guard as the repo listing: a gateway that ignores
|
||||
// offset hands back the identical full page forever.
|
||||
firstID := resp.Data[0].ID
|
||||
if firstID != 0 && firstID == lastFirstID {
|
||||
logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)
|
||||
return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)
|
||||
}
|
||||
lastFirstID = firstID
|
||||
offset += defaultPageSize
|
||||
}
|
||||
return all, nil
|
||||
|
|
|
|||
|
|
@ -0,0 +1,299 @@
|
|||
package yuque
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// paginationHopCap mirrors maxPaginationHops in client.go. It is spelled as a
|
||||
// literal so this file also compiles — and then fails on behaviour — against a
|
||||
// tree that has no guard at all.
|
||||
const paginationHopCap = 10000
|
||||
|
||||
// paginationBudget is how many requests the "always a full page" fakes below
|
||||
// serve before answering 400. A walk that lacks a guard therefore fails on the
|
||||
// budget error after paginationBudget+1 requests (and on the request-count
|
||||
// assertion) instead of spinning until the go test timeout.
|
||||
const paginationBudget = 300
|
||||
|
||||
// fullDocPage builds a full-ish page of docs with ids startID..startID+n-1.
|
||||
func fullDocPage(startID int64, n int) []v2Doc {
|
||||
docs := make([]v2Doc, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
docs = append(docs, v2Doc{
|
||||
ID: startID + int64(i),
|
||||
Type: "Doc",
|
||||
Status: "1",
|
||||
Title: fmt.Sprintf("D%d", startID+int64(i)),
|
||||
ContentUpdatedAt: "2026-04-20T00:00:00Z",
|
||||
})
|
||||
}
|
||||
return docs
|
||||
}
|
||||
|
||||
// fullRepoPage builds a full-ish page of repos with ids startID..startID+n-1.
|
||||
func fullRepoPage(startID int64, n int) []v2Repo {
|
||||
repos := make([]v2Repo, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
repos = append(repos, v2Repo{
|
||||
ID: startID + int64(i),
|
||||
Type: "Book",
|
||||
Namespace: fmt.Sprintf("alice/book-%d", startID+int64(i)),
|
||||
})
|
||||
}
|
||||
return repos
|
||||
}
|
||||
|
||||
// minimalPage renders a page of defaultPageSize id-only entries, so the hop cap
|
||||
// test can serve 10000 pages without materialising gigabytes of JSON.
|
||||
func minimalPage(prefix string, startID int64) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(`{"data":[`)
|
||||
for i := 0; i < defaultPageSize; i++ {
|
||||
if i > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
fmt.Fprintf(&sb, `{"%s":%d}`, prefix, startID+int64(i))
|
||||
}
|
||||
sb.WriteString(`]}`)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// alwaysFullPageServer answers every offset with the same full page: a gateway
|
||||
// or self-hosted deployment that ignores `offset`. Without a repeated-page
|
||||
// guard the walk keeps appending identical entries.
|
||||
func alwaysFullPageServer(t *testing.T, body interface{}) (*httptest.Server, *atomic.Int64) {
|
||||
t.Helper()
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if requests.Add(1) > paginationBudget {
|
||||
http.Error(w, "pagination guard missing: too many requests", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server, &requests
|
||||
}
|
||||
|
||||
// A docs listing whose meta.total is already covered by the first page must end
|
||||
// there. The fake keeps answering a full page for every offset, so before the
|
||||
// fix the walk ran until the fake's budget cut it off 300 requests later.
|
||||
func TestListBookDocs_StopsAtMetaTotal(t *testing.T) {
|
||||
resp := v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}
|
||||
resp.Meta.Total = 5 // far below what this page alone hands out
|
||||
server, requests := alwaysFullPageServer(t, resp)
|
||||
|
||||
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
|
||||
docs, err := c.ListBookDocs(context.Background(), 4242)
|
||||
|
||||
require.NoError(t, err, "meta.total covered by the first page must end the walk")
|
||||
require.Len(t, docs, defaultPageSize)
|
||||
require.EqualValues(t, 1, requests.Load(),
|
||||
"the walk must stop on the page that reaches meta.total")
|
||||
}
|
||||
|
||||
// A full page whose first id repeats the previous page means the offset walk is
|
||||
// not making progress. Each endpoint must report that instead of appending the
|
||||
// same page forever.
|
||||
func TestPaginationStopsOnRepeatedPage(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
call func(context.Context, *client) (int, error)
|
||||
}{
|
||||
{
|
||||
name: "book docs without meta.total",
|
||||
path: "/api/v2/repos/4242/docs",
|
||||
call: func(ctx context.Context, c *client) (int, error) {
|
||||
docs, err := c.ListBookDocs(ctx, 4242)
|
||||
return len(docs), err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user repos",
|
||||
path: "/api/v2/users/alice/repos",
|
||||
call: func(ctx context.Context, c *client) (int, error) {
|
||||
repos, err := c.ListUserRepos(ctx, "alice")
|
||||
return len(repos), err
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var body interface{}
|
||||
if strings.HasSuffix(tc.path, "/docs") {
|
||||
body = v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}
|
||||
} else {
|
||||
body = v2RepoListResponse{Data: fullRepoPage(1, defaultPageSize)}
|
||||
}
|
||||
server, requests := alwaysFullPageServer(t, body)
|
||||
|
||||
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
|
||||
n, err := tc.call(context.Background(), c)
|
||||
|
||||
require.Error(t, err, "a repeated full page must not be followed forever")
|
||||
require.Contains(t, err.Error(), "repeated page")
|
||||
require.Zero(t, n, "a stalled walk must not hand back a partial listing")
|
||||
require.LessOrEqual(t, requests.Load(), int64(2),
|
||||
"the guard must reject the repeated page on the second request")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A server that never repeats an id but also never returns a short page can only
|
||||
// be stopped by the hop cap. It must be reported, and the log must carry it too:
|
||||
// visually this failure looks like a healthy, still-running sync.
|
||||
func TestPaginationHopCapStopsWalk(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
call func(context.Context, *client) error
|
||||
}{
|
||||
{
|
||||
name: "book docs",
|
||||
call: func(ctx context.Context, c *client) error {
|
||||
_, err := c.ListBookDocs(ctx, 4242)
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user repos",
|
||||
call: func(ctx context.Context, c *client) error {
|
||||
_, err := c.ListUserRepos(ctx, "alice")
|
||||
return err
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := requests.Add(1)
|
||||
if n > paginationHopCap+10 {
|
||||
http.Error(w, "hop cap missing: too many requests", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
offset, err := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
if err != nil {
|
||||
http.Error(w, "unexpected offset", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Fresh ids on every page and no meta.total, so only the hop cap
|
||||
// can end the walk.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(minimalPage("id", int64(offset)+1)))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
|
||||
|
||||
var logs bytes.Buffer
|
||||
logger.SetOutput(&logs)
|
||||
t.Cleanup(logger.ConfigureFromEnv)
|
||||
|
||||
err := tc.call(context.Background(), c)
|
||||
|
||||
require.ErrorContains(t, err, "pagination exceeded 10000 pages")
|
||||
require.EqualValues(t, paginationHopCap, requests.Load())
|
||||
require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
|
||||
"hitting the hop cap must leave a warning in the log")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pagedServer serves `total` items in defaultPageSize-sized offset pages, with
|
||||
// the docs response advertising meta.total. It is the well-behaved listing the
|
||||
// guards must not disturb.
|
||||
func pagedServer(t *testing.T, kind string, total int) (*httptest.Server, *atomic.Int64) {
|
||||
t.Helper()
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
offset, err := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
if err != nil {
|
||||
http.Error(w, "unexpected offset", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
size := total - offset
|
||||
if size > defaultPageSize {
|
||||
size = defaultPageSize
|
||||
}
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if kind == "docs" {
|
||||
resp := v2DocListResponse{Data: fullDocPage(int64(offset)+1, size)}
|
||||
resp.Meta.Total = total
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(v2RepoListResponse{Data: fullRepoPage(int64(offset)+1, size)})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server, &requests
|
||||
}
|
||||
|
||||
// The guards must not truncate a listing that genuinely spans several pages.
|
||||
func TestPaginationCollectsRealPages(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
kind string
|
||||
total int
|
||||
wantReqs int64
|
||||
call func(context.Context, *client) (int, error)
|
||||
}{
|
||||
{
|
||||
name: "docs reaching meta.total exactly",
|
||||
kind: "docs",
|
||||
total: 2 * defaultPageSize,
|
||||
wantReqs: 2,
|
||||
call: func(ctx context.Context, c *client) (int, error) {
|
||||
docs, err := c.ListBookDocs(ctx, 4242)
|
||||
return len(docs), err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "docs with a short final page",
|
||||
kind: "docs",
|
||||
total: defaultPageSize + 30,
|
||||
wantReqs: 2,
|
||||
call: func(ctx context.Context, c *client) (int, error) {
|
||||
docs, err := c.ListBookDocs(ctx, 4242)
|
||||
return len(docs), err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "repos with a short final page",
|
||||
kind: "repos",
|
||||
total: defaultPageSize + 30,
|
||||
wantReqs: 2,
|
||||
call: func(ctx context.Context, c *client) (int, error) {
|
||||
repos, err := c.ListUserRepos(ctx, "alice")
|
||||
return len(repos), err
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server, requests := pagedServer(t, tc.kind, tc.total)
|
||||
|
||||
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
|
||||
got, err := tc.call(context.Background(), c)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.total, got, "every item of a real multi-page listing must survive")
|
||||
require.EqualValues(t, tc.wantReqs, requests.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue