fix(yuque): 两处 offset 分页补跳数上限、重复页检测与 meta.total 提前结束 (#4415)

main
linus-liu-web 2026-10-17 23:17:11 +08:00 committed by GitHub
parent 073b5ea616
commit 4d767d9dc4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 342 additions and 2 deletions

View File

@ -23,6 +23,15 @@ const (
// maxJSONResponseBytes bounds every API response body. Listings and document
// payloads are small; a larger body means a broken or hostile server.
maxJSONResponseBytes = 16 << 20
// maxPaginationHops bounds every offset-paginated walk in this file. The
// only natural exit is a short page, which a gateway, CDN, or self-hosted
// deployment that answers every offset with a full page never produces;
// /users/{login}/repos carries no meta.total to stop on either. Without the
// cap such a walk keeps calling the API until the sync task hits its
// deadline, with `all` growing all the way. Mirrors the guard Notion,
// Confluence, GitLab, and IMA already carry.
maxPaginationHops = 10000
)
// client wraps the Yuque Open API.
@ -273,7 +282,12 @@ func (c *client) ListGroupRepos(ctx context.Context, login string) ([]v2Repo, er
func (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2Repo, error) {
var all []v2Repo
offset := 0
for {
lastFirstID := int64(0)
for hop := 0; ; hop++ {
if hop >= maxPaginationHops {
logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)
return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)
}
q := buildQuery(map[string]string{
"type": "Book",
"offset": fmt.Sprintf("%d", offset),
@ -287,6 +301,15 @@ func (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2R
if len(resp.Data) < defaultPageSize {
break
}
// A full page whose first id repeats the previous page means the server
// is answering the same page for a growing offset; following it would
// append the same repos forever.
firstID := resp.Data[0].ID
if firstID != 0 && firstID == lastFirstID {
logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)
return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)
}
lastFirstID = firstID
offset += defaultPageSize
}
return all, nil
@ -298,7 +321,12 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, error
basePath := fmt.Sprintf("/api/v2/repos/%d/docs", bookID)
var all []v2Doc
offset := 0
for {
lastFirstID := int64(0)
for hop := 0; ; hop++ {
if hop >= maxPaginationHops {
logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)
return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)
}
q := buildQuery(map[string]string{
"offset": fmt.Sprintf("%d", offset),
"limit": fmt.Sprintf("%d", defaultPageSize),
@ -311,6 +339,19 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, error
if len(resp.Data) < defaultPageSize {
break
}
// meta.total counts the whole listing, so a page that reaches it is the
// last one; asking for the next offset would only fetch an empty page.
if resp.Meta.Total > 0 && offset+len(resp.Data) >= resp.Meta.Total {
break
}
// Same repeated-page guard as the repo listing: a gateway that ignores
// offset hands back the identical full page forever.
firstID := resp.Data[0].ID
if firstID != 0 && firstID == lastFirstID {
logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)
return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)
}
lastFirstID = firstID
offset += defaultPageSize
}
return all, nil

View File

@ -0,0 +1,299 @@
package yuque
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync/atomic"
"testing"
"github.com/stretchr/testify/require"
"github.com/Tencent/WeKnora/internal/logger"
)
// paginationHopCap mirrors maxPaginationHops in client.go. It is spelled as a
// literal so this file also compiles — and then fails on behaviour — against a
// tree that has no guard at all.
const paginationHopCap = 10000
// paginationBudget is how many requests the "always a full page" fakes below
// serve before answering 400. A walk that lacks a guard therefore fails on the
// budget error after paginationBudget+1 requests (and on the request-count
// assertion) instead of spinning until the go test timeout.
const paginationBudget = 300
// fullDocPage builds a full-ish page of docs with ids startID..startID+n-1.
func fullDocPage(startID int64, n int) []v2Doc {
docs := make([]v2Doc, 0, n)
for i := 0; i < n; i++ {
docs = append(docs, v2Doc{
ID: startID + int64(i),
Type: "Doc",
Status: "1",
Title: fmt.Sprintf("D%d", startID+int64(i)),
ContentUpdatedAt: "2026-04-20T00:00:00Z",
})
}
return docs
}
// fullRepoPage builds a full-ish page of repos with ids startID..startID+n-1.
func fullRepoPage(startID int64, n int) []v2Repo {
repos := make([]v2Repo, 0, n)
for i := 0; i < n; i++ {
repos = append(repos, v2Repo{
ID: startID + int64(i),
Type: "Book",
Namespace: fmt.Sprintf("alice/book-%d", startID+int64(i)),
})
}
return repos
}
// minimalPage renders a page of defaultPageSize id-only entries, so the hop cap
// test can serve 10000 pages without materialising gigabytes of JSON.
func minimalPage(prefix string, startID int64) string {
var sb strings.Builder
sb.WriteString(`{"data":[`)
for i := 0; i < defaultPageSize; i++ {
if i > 0 {
sb.WriteByte(',')
}
fmt.Fprintf(&sb, `{"%s":%d}`, prefix, startID+int64(i))
}
sb.WriteString(`]}`)
return sb.String()
}
// alwaysFullPageServer answers every offset with the same full page: a gateway
// or self-hosted deployment that ignores `offset`. Without a repeated-page
// guard the walk keeps appending identical entries.
func alwaysFullPageServer(t *testing.T, body interface{}) (*httptest.Server, *atomic.Int64) {
t.Helper()
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if requests.Add(1) > paginationBudget {
http.Error(w, "pagination guard missing: too many requests", http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(body)
}))
t.Cleanup(server.Close)
return server, &requests
}
// A docs listing whose meta.total is already covered by the first page must end
// there. The fake keeps answering a full page for every offset, so before the
// fix the walk ran until the fake's budget cut it off 300 requests later.
func TestListBookDocs_StopsAtMetaTotal(t *testing.T) {
resp := v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}
resp.Meta.Total = 5 // far below what this page alone hands out
server, requests := alwaysFullPageServer(t, resp)
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
docs, err := c.ListBookDocs(context.Background(), 4242)
require.NoError(t, err, "meta.total covered by the first page must end the walk")
require.Len(t, docs, defaultPageSize)
require.EqualValues(t, 1, requests.Load(),
"the walk must stop on the page that reaches meta.total")
}
// A full page whose first id repeats the previous page means the offset walk is
// not making progress. Each endpoint must report that instead of appending the
// same page forever.
func TestPaginationStopsOnRepeatedPage(t *testing.T) {
for _, tc := range []struct {
name string
path string
call func(context.Context, *client) (int, error)
}{
{
name: "book docs without meta.total",
path: "/api/v2/repos/4242/docs",
call: func(ctx context.Context, c *client) (int, error) {
docs, err := c.ListBookDocs(ctx, 4242)
return len(docs), err
},
},
{
name: "user repos",
path: "/api/v2/users/alice/repos",
call: func(ctx context.Context, c *client) (int, error) {
repos, err := c.ListUserRepos(ctx, "alice")
return len(repos), err
},
},
} {
t.Run(tc.name, func(t *testing.T) {
var body interface{}
if strings.HasSuffix(tc.path, "/docs") {
body = v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}
} else {
body = v2RepoListResponse{Data: fullRepoPage(1, defaultPageSize)}
}
server, requests := alwaysFullPageServer(t, body)
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
n, err := tc.call(context.Background(), c)
require.Error(t, err, "a repeated full page must not be followed forever")
require.Contains(t, err.Error(), "repeated page")
require.Zero(t, n, "a stalled walk must not hand back a partial listing")
require.LessOrEqual(t, requests.Load(), int64(2),
"the guard must reject the repeated page on the second request")
})
}
}
// A server that never repeats an id but also never returns a short page can only
// be stopped by the hop cap. It must be reported, and the log must carry it too:
// visually this failure looks like a healthy, still-running sync.
func TestPaginationHopCapStopsWalk(t *testing.T) {
for _, tc := range []struct {
name string
call func(context.Context, *client) error
}{
{
name: "book docs",
call: func(ctx context.Context, c *client) error {
_, err := c.ListBookDocs(ctx, 4242)
return err
},
},
{
name: "user repos",
call: func(ctx context.Context, c *client) error {
_, err := c.ListUserRepos(ctx, "alice")
return err
},
},
} {
t.Run(tc.name, func(t *testing.T) {
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := requests.Add(1)
if n > paginationHopCap+10 {
http.Error(w, "hop cap missing: too many requests", http.StatusBadRequest)
return
}
offset, err := strconv.Atoi(r.URL.Query().Get("offset"))
if err != nil {
http.Error(w, "unexpected offset", http.StatusBadRequest)
return
}
// Fresh ids on every page and no meta.total, so only the hop cap
// can end the walk.
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(minimalPage("id", int64(offset)+1)))
}))
t.Cleanup(server.Close)
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
var logs bytes.Buffer
logger.SetOutput(&logs)
t.Cleanup(logger.ConfigureFromEnv)
err := tc.call(context.Background(), c)
require.ErrorContains(t, err, "pagination exceeded 10000 pages")
require.EqualValues(t, paginationHopCap, requests.Load())
require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
"hitting the hop cap must leave a warning in the log")
})
}
}
// pagedServer serves `total` items in defaultPageSize-sized offset pages, with
// the docs response advertising meta.total. It is the well-behaved listing the
// guards must not disturb.
func pagedServer(t *testing.T, kind string, total int) (*httptest.Server, *atomic.Int64) {
t.Helper()
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
offset, err := strconv.Atoi(r.URL.Query().Get("offset"))
if err != nil {
http.Error(w, "unexpected offset", http.StatusBadRequest)
return
}
size := total - offset
if size > defaultPageSize {
size = defaultPageSize
}
if size < 0 {
size = 0
}
w.Header().Set("Content-Type", "application/json")
if kind == "docs" {
resp := v2DocListResponse{Data: fullDocPage(int64(offset)+1, size)}
resp.Meta.Total = total
_ = json.NewEncoder(w).Encode(resp)
return
}
_ = json.NewEncoder(w).Encode(v2RepoListResponse{Data: fullRepoPage(int64(offset)+1, size)})
}))
t.Cleanup(server.Close)
return server, &requests
}
// The guards must not truncate a listing that genuinely spans several pages.
func TestPaginationCollectsRealPages(t *testing.T) {
for _, tc := range []struct {
name string
kind string
total int
wantReqs int64
call func(context.Context, *client) (int, error)
}{
{
name: "docs reaching meta.total exactly",
kind: "docs",
total: 2 * defaultPageSize,
wantReqs: 2,
call: func(ctx context.Context, c *client) (int, error) {
docs, err := c.ListBookDocs(ctx, 4242)
return len(docs), err
},
},
{
name: "docs with a short final page",
kind: "docs",
total: defaultPageSize + 30,
wantReqs: 2,
call: func(ctx context.Context, c *client) (int, error) {
docs, err := c.ListBookDocs(ctx, 4242)
return len(docs), err
},
},
{
name: "repos with a short final page",
kind: "repos",
total: defaultPageSize + 30,
wantReqs: 2,
call: func(ctx context.Context, c *client) (int, error) {
repos, err := c.ListUserRepos(ctx, "alice")
return len(repos), err
},
},
} {
t.Run(tc.name, func(t *testing.T) {
server, requests := pagedServer(t, tc.kind, tc.total)
c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})
got, err := tc.call(context.Background(), c)
require.NoError(t, err)
require.Equal(t, tc.total, got, "every item of a real multi-page listing must survive")
require.EqualValues(t, tc.wantReqs, requests.Load())
})
}
}