fix(gitlab): 分页循环补重复页防护与跳数上限 (#3841)

projects() 与 tree() 只依据服务端返回的 X-Next-Page 判断是否继续,
该值重复或不前进时循环不会结束:同一页被反复请求、结果切片持续
增长,只能等上层 context 到期。

两条循环改用 trackPage:页码已请求过即返回错误,不静默当作读完;
页数达到 maxPaginationHops 时记警告并返回错误。正常翻页路径不变。

Co-authored-by: linus-liu-web <11640009+linus-liu-web@users.noreply.github.com>
main
linus-liu-web 2026-10-08 01:21:37 +08:00 committed by GitHub
parent b5c74b2dd6
commit a2aeb97a01
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 176 additions and 0 deletions

View File

@ -14,6 +14,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
)
const (
@ -39,6 +40,11 @@ func base64FileBytes(rawLimit int64) int64 {
return rawLimit/3*4 + (1 << 20)
}
// maxPaginationHops bounds a single paginated walk. GitLab reports the next
// page through the X-Next-Page header, so a broken instance or reverse proxy
// could keep advertising pages indefinitely.
const maxPaginationHops = 10000
type client struct {
baseURL, token string
http *http.Client
@ -206,13 +212,33 @@ func (c *client) project(ctx context.Context, id string) (*project, error) {
return &p, err
}
// trackPage records the page a paginated walk is about to request. A page that
// was already requested means X-Next-Page is not advancing, so the walk stops
// with an error instead of re-reading the same page; the hop cap bounds a
// server that keeps advertising fresh page numbers.
func trackPage(ctx context.Context, scope string, seen map[string]struct{}, page string) error {
if _, dup := seen[page]; dup {
return fmt.Errorf("gitlab %s pagination repeated page %q", scope, page)
}
if len(seen) >= maxPaginationHops {
logger.Warnf(ctx, "[GitLab] %s pagination exceeded %d pages; aborting", scope, maxPaginationHops)
return fmt.Errorf("gitlab %s pagination exceeded %d pages", scope, maxPaginationHops)
}
seen[page] = struct{}{}
return nil
}
func (c *client) projects(ctx context.Context) ([]project, error) {
q := url.Values{
"membership": {"true"}, "per_page": {"100"}, "page": {"1"},
"order_by": {"path_with_namespace"}, "sort": {"asc"},
}
var all []project
seen := make(map[string]struct{})
for {
if err := trackPage(ctx, "project", seen, q.Get("page")); err != nil {
return nil, err
}
var page []project
nextPage, err := c.getPage(ctx, "/projects", q, &page)
if err != nil {
@ -251,7 +277,11 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er
}
endpoint := "/projects/" + projectPath(id) + "/repository/tree"
var all []treeEntry
seen := make(map[string]struct{})
for {
if err := trackPage(ctx, "tree", seen, q.Get("page")); err != nil {
return nil, err
}
var page []treeEntry
nextPage, err := c.getPage(ctx, endpoint, q, &page)
if err != nil {

View File

@ -0,0 +1,146 @@
package gitlab
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/stretchr/testify/require"
)
// guardTestTimeout bounds every walk here so that a missing guard fails the
// test instead of hanging it.
const guardTestTimeout = 5 * time.Second
// repeatedPageServer always advertises the same X-Next-Page value and refuses
// to serve more than three requests: a walk without a repeated-page guard then
// fails quickly with an unexpected error instead of spinning until the context
// deadline.
func repeatedPageServer(t *testing.T) (*httptest.Server, *int64) {
t.Helper()
allowLocalGitLabServer(t)
var requests int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if atomic.AddInt64(&requests, 1) > 3 {
http.Error(w, "pagination guard missing: too many requests", http.StatusTooManyRequests)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Next-Page", "2")
if strings.HasSuffix(r.URL.Path, "/repository/tree") {
_, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`))
return
}
_, _ = w.Write([]byte(`[{"id":1,"path_with_namespace":"group/project-001"}]`))
}))
t.Cleanup(server.Close)
return server, &requests
}
func TestProjectsStopsOnRepeatedNextPage(t *testing.T) {
server, requests := repeatedPageServer(t)
c, err := newClient(server.URL, "test-token")
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
defer cancel()
projects, err := c.projects(ctx)
require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
require.Nil(t, projects)
require.Contains(t, err.Error(), `gitlab project pagination repeated page "2"`)
require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
}
func TestTreeStopsOnRepeatedNextPage(t *testing.T) {
server, requests := repeatedPageServer(t)
c, err := newClient(server.URL, "test-token")
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
defer cancel()
entries, err := c.tree(ctx, "1", "main", "")
require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
require.Nil(t, entries)
require.Contains(t, err.Error(), `gitlab tree pagination repeated page "2"`)
require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
}
// TestPaginationHopCapStopsWalk drives each walk with a server that always
// advertises a fresh page number, so only the hop cap can end it.
func TestPaginationHopCapStopsWalk(t *testing.T) {
for _, tc := range []struct {
name string
want string
walk func(context.Context, *client) error
}{
{
name: "projects",
want: "gitlab project pagination exceeded 10000 pages",
walk: func(ctx context.Context, c *client) error {
_, err := c.projects(ctx)
return err
},
},
{
name: "tree",
want: "gitlab tree pagination exceeded 10000 pages",
walk: func(ctx context.Context, c *client) error {
_, err := c.tree(ctx, "1", "main", "")
return err
},
},
} {
t.Run(tc.name, func(t *testing.T) {
allowLocalGitLabServer(t)
var requests int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt64(&requests, 1)
if n > maxPaginationHops+10 {
http.Error(w, "hop cap missing: too many requests", http.StatusTooManyRequests)
return
}
page, convErr := strconv.Atoi(r.URL.Query().Get("page"))
if convErr != nil {
http.Error(w, "unexpected page", http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Next-Page", strconv.Itoa(page+1))
_, _ = w.Write([]byte(`[]`))
}))
t.Cleanup(server.Close)
c, err := newClient(server.URL, "test-token")
require.NoError(t, err)
var logs bytes.Buffer
logger.SetOutput(&logs)
t.Cleanup(logger.ConfigureFromEnv)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
err = tc.walk(ctx, c)
require.EqualError(t, err, tc.want)
require.Equal(t, int64(maxPaginationHops), atomic.LoadInt64(&requests))
require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
"hitting the cap must leave a warning in the log")
})
}
}
func TestTrackPageAllowsForwardPages(t *testing.T) {
seen := make(map[string]struct{})
for _, page := range []string{"1", "2", "3"} {
require.NoError(t, trackPage(context.Background(), "project", seen, page))
}
require.Error(t, trackPage(context.Background(), "project", seen, "2"))
}