fix(gitlab): 分页循环补重复页防护与跳数上限 (#3841)
projects() 与 tree() 只依据服务端返回的 X-Next-Page 判断是否继续, 该值重复或不前进时循环不会结束:同一页被反复请求、结果切片持续 增长,只能等上层 context 到期。 两条循环改用 trackPage:页码已请求过即返回错误,不静默当作读完; 页数达到 maxPaginationHops 时记警告并返回错误。正常翻页路径不变。 Co-authored-by: linus-liu-web <11640009+linus-liu-web@users.noreply.github.com>main
parent
b5c74b2dd6
commit
a2aeb97a01
|
|
@ -14,6 +14,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -39,6 +40,11 @@ func base64FileBytes(rawLimit int64) int64 {
|
|||
return rawLimit/3*4 + (1 << 20)
|
||||
}
|
||||
|
||||
// maxPaginationHops bounds a single paginated walk. GitLab reports the next
|
||||
// page through the X-Next-Page header, so a broken instance or reverse proxy
|
||||
// could keep advertising pages indefinitely.
|
||||
const maxPaginationHops = 10000
|
||||
|
||||
type client struct {
|
||||
baseURL, token string
|
||||
http *http.Client
|
||||
|
|
@ -206,13 +212,33 @@ func (c *client) project(ctx context.Context, id string) (*project, error) {
|
|||
return &p, err
|
||||
}
|
||||
|
||||
// trackPage records the page a paginated walk is about to request. A page that
|
||||
// was already requested means X-Next-Page is not advancing, so the walk stops
|
||||
// with an error instead of re-reading the same page; the hop cap bounds a
|
||||
// server that keeps advertising fresh page numbers.
|
||||
func trackPage(ctx context.Context, scope string, seen map[string]struct{}, page string) error {
|
||||
if _, dup := seen[page]; dup {
|
||||
return fmt.Errorf("gitlab %s pagination repeated page %q", scope, page)
|
||||
}
|
||||
if len(seen) >= maxPaginationHops {
|
||||
logger.Warnf(ctx, "[GitLab] %s pagination exceeded %d pages; aborting", scope, maxPaginationHops)
|
||||
return fmt.Errorf("gitlab %s pagination exceeded %d pages", scope, maxPaginationHops)
|
||||
}
|
||||
seen[page] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) projects(ctx context.Context) ([]project, error) {
|
||||
q := url.Values{
|
||||
"membership": {"true"}, "per_page": {"100"}, "page": {"1"},
|
||||
"order_by": {"path_with_namespace"}, "sort": {"asc"},
|
||||
}
|
||||
var all []project
|
||||
seen := make(map[string]struct{})
|
||||
for {
|
||||
if err := trackPage(ctx, "project", seen, q.Get("page")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page []project
|
||||
nextPage, err := c.getPage(ctx, "/projects", q, &page)
|
||||
if err != nil {
|
||||
|
|
@ -251,7 +277,11 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er
|
|||
}
|
||||
endpoint := "/projects/" + projectPath(id) + "/repository/tree"
|
||||
var all []treeEntry
|
||||
seen := make(map[string]struct{})
|
||||
for {
|
||||
if err := trackPage(ctx, "tree", seen, q.Get("page")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page []treeEntry
|
||||
nextPage, err := c.getPage(ctx, endpoint, q, &page)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
package gitlab
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// guardTestTimeout bounds every walk here so that a missing guard fails the
|
||||
// test instead of hanging it.
|
||||
const guardTestTimeout = 5 * time.Second
|
||||
|
||||
// repeatedPageServer always advertises the same X-Next-Page value and refuses
|
||||
// to serve more than three requests: a walk without a repeated-page guard then
|
||||
// fails quickly with an unexpected error instead of spinning until the context
|
||||
// deadline.
|
||||
func repeatedPageServer(t *testing.T) (*httptest.Server, *int64) {
|
||||
t.Helper()
|
||||
allowLocalGitLabServer(t)
|
||||
var requests int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if atomic.AddInt64(&requests, 1) > 3 {
|
||||
http.Error(w, "pagination guard missing: too many requests", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("X-Next-Page", "2")
|
||||
if strings.HasSuffix(r.URL.Path, "/repository/tree") {
|
||||
_, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"id":1,"path_with_namespace":"group/project-001"}]`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server, &requests
|
||||
}
|
||||
|
||||
func TestProjectsStopsOnRepeatedNextPage(t *testing.T) {
|
||||
server, requests := repeatedPageServer(t)
|
||||
c, err := newClient(server.URL, "test-token")
|
||||
require.NoError(t, err)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
projects, err := c.projects(ctx)
|
||||
|
||||
require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
|
||||
require.Nil(t, projects)
|
||||
require.Contains(t, err.Error(), `gitlab project pagination repeated page "2"`)
|
||||
require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
|
||||
}
|
||||
|
||||
func TestTreeStopsOnRepeatedNextPage(t *testing.T) {
|
||||
server, requests := repeatedPageServer(t)
|
||||
c, err := newClient(server.URL, "test-token")
|
||||
require.NoError(t, err)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
entries, err := c.tree(ctx, "1", "main", "")
|
||||
|
||||
require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
|
||||
require.Nil(t, entries)
|
||||
require.Contains(t, err.Error(), `gitlab tree pagination repeated page "2"`)
|
||||
require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
|
||||
}
|
||||
|
||||
// TestPaginationHopCapStopsWalk drives each walk with a server that always
|
||||
// advertises a fresh page number, so only the hop cap can end it.
|
||||
func TestPaginationHopCapStopsWalk(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
want string
|
||||
walk func(context.Context, *client) error
|
||||
}{
|
||||
{
|
||||
name: "projects",
|
||||
want: "gitlab project pagination exceeded 10000 pages",
|
||||
walk: func(ctx context.Context, c *client) error {
|
||||
_, err := c.projects(ctx)
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "tree",
|
||||
want: "gitlab tree pagination exceeded 10000 pages",
|
||||
walk: func(ctx context.Context, c *client) error {
|
||||
_, err := c.tree(ctx, "1", "main", "")
|
||||
return err
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
allowLocalGitLabServer(t)
|
||||
var requests int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt64(&requests, 1)
|
||||
if n > maxPaginationHops+10 {
|
||||
http.Error(w, "hop cap missing: too many requests", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
page, convErr := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if convErr != nil {
|
||||
http.Error(w, "unexpected page", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("X-Next-Page", strconv.Itoa(page+1))
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
c, err := newClient(server.URL, "test-token")
|
||||
require.NoError(t, err)
|
||||
|
||||
var logs bytes.Buffer
|
||||
logger.SetOutput(&logs)
|
||||
t.Cleanup(logger.ConfigureFromEnv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
err = tc.walk(ctx, c)
|
||||
|
||||
require.EqualError(t, err, tc.want)
|
||||
require.Equal(t, int64(maxPaginationHops), atomic.LoadInt64(&requests))
|
||||
require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
|
||||
"hitting the cap must leave a warning in the log")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackPageAllowsForwardPages(t *testing.T) {
|
||||
seen := make(map[string]struct{})
|
||||
for _, page := range []string{"1", "2", "3"} {
|
||||
require.NoError(t, trackPage(context.Background(), "project", seen, page))
|
||||
}
|
||||
require.Error(t, trackPage(context.Background(), "project", seen, "2"))
|
||||
}
|
||||
Loading…
Reference in New Issue