From a2aeb97a01203b23b6093eba2369583b460aaa66 Mon Sep 17 00:00:00 2001 From: linus-liu-web <395141741@qq.com> Date: Thu, 8 Oct 2026 01:21:37 +0800 Subject: [PATCH] =?UTF-8?q?fix(gitlab):=20=E5=88=86=E9=A1=B5=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E8=A1=A5=E9=87=8D=E5=A4=8D=E9=A1=B5=E9=98=B2=E6=8A=A4?= =?UTF-8?q?=E4=B8=8E=E8=B7=B3=E6=95=B0=E4=B8=8A=E9=99=90=20(#3841)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit projects() 与 tree() 只依据服务端返回的 X-Next-Page 判断是否继续, 该值重复或不前进时循环不会结束:同一页被反复请求、结果切片持续 增长,只能等上层 context 到期。 两条循环改用 trackPage:页码已请求过即返回错误,不静默当作读完; 页数达到 maxPaginationHops 时记警告并返回错误。正常翻页路径不变。 Co-authored-by: linus-liu-web <11640009+linus-liu-web@users.noreply.github.com> --- .../datasource/connector/gitlab/client.go | 30 ++++ .../connector/gitlab/pagination_guard_test.go | 146 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 internal/datasource/connector/gitlab/pagination_guard_test.go diff --git a/internal/datasource/connector/gitlab/client.go b/internal/datasource/connector/gitlab/client.go index 5a26de62..3ebefb94 100644 --- a/internal/datasource/connector/gitlab/client.go +++ b/internal/datasource/connector/gitlab/client.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" ) const ( @@ -39,6 +40,11 @@ func base64FileBytes(rawLimit int64) int64 { return rawLimit/3*4 + (1 << 20) } +// maxPaginationHops bounds a single paginated walk. GitLab reports the next +// page through the X-Next-Page header, so a broken instance or reverse proxy +// could keep advertising pages indefinitely. +const maxPaginationHops = 10000 + type client struct { baseURL, token string http *http.Client @@ -206,13 +212,33 @@ func (c *client) project(ctx context.Context, id string) (*project, error) { return &p, err } +// trackPage records the page a paginated walk is about to request. A page that +// was already requested means X-Next-Page is not advancing, so the walk stops +// with an error instead of re-reading the same page; the hop cap bounds a +// server that keeps advertising fresh page numbers. +func trackPage(ctx context.Context, scope string, seen map[string]struct{}, page string) error { + if _, dup := seen[page]; dup { + return fmt.Errorf("gitlab %s pagination repeated page %q", scope, page) + } + if len(seen) >= maxPaginationHops { + logger.Warnf(ctx, "[GitLab] %s pagination exceeded %d pages; aborting", scope, maxPaginationHops) + return fmt.Errorf("gitlab %s pagination exceeded %d pages", scope, maxPaginationHops) + } + seen[page] = struct{}{} + return nil +} + func (c *client) projects(ctx context.Context) ([]project, error) { q := url.Values{ "membership": {"true"}, "per_page": {"100"}, "page": {"1"}, "order_by": {"path_with_namespace"}, "sort": {"asc"}, } var all []project + seen := make(map[string]struct{}) for { + if err := trackPage(ctx, "project", seen, q.Get("page")); err != nil { + return nil, err + } var page []project nextPage, err := c.getPage(ctx, "/projects", q, &page) if err != nil { @@ -251,7 +277,11 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er } endpoint := "/projects/" + projectPath(id) + "/repository/tree" var all []treeEntry + seen := make(map[string]struct{}) for { + if err := trackPage(ctx, "tree", seen, q.Get("page")); err != nil { + return nil, err + } var page []treeEntry nextPage, err := c.getPage(ctx, endpoint, q, &page) if err != nil { diff --git a/internal/datasource/connector/gitlab/pagination_guard_test.go b/internal/datasource/connector/gitlab/pagination_guard_test.go new file mode 100644 index 00000000..651e9571 --- /dev/null +++ b/internal/datasource/connector/gitlab/pagination_guard_test.go @@ -0,0 +1,146 @@ +package gitlab + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/stretchr/testify/require" +) + +// guardTestTimeout bounds every walk here so that a missing guard fails the +// test instead of hanging it. +const guardTestTimeout = 5 * time.Second + +// repeatedPageServer always advertises the same X-Next-Page value and refuses +// to serve more than three requests: a walk without a repeated-page guard then +// fails quickly with an unexpected error instead of spinning until the context +// deadline. +func repeatedPageServer(t *testing.T) (*httptest.Server, *int64) { + t.Helper() + allowLocalGitLabServer(t) + var requests int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt64(&requests, 1) > 3 { + http.Error(w, "pagination guard missing: too many requests", http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Next-Page", "2") + if strings.HasSuffix(r.URL.Path, "/repository/tree") { + _, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`)) + return + } + _, _ = w.Write([]byte(`[{"id":1,"path_with_namespace":"group/project-001"}]`)) + })) + t.Cleanup(server.Close) + return server, &requests +} + +func TestProjectsStopsOnRepeatedNextPage(t *testing.T) { + server, requests := repeatedPageServer(t) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout) + defer cancel() + + projects, err := c.projects(ctx) + + require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list") + require.Nil(t, projects) + require.Contains(t, err.Error(), `gitlab project pagination repeated page "2"`) + require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page") +} + +func TestTreeStopsOnRepeatedNextPage(t *testing.T) { + server, requests := repeatedPageServer(t) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout) + defer cancel() + + entries, err := c.tree(ctx, "1", "main", "") + + require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list") + require.Nil(t, entries) + require.Contains(t, err.Error(), `gitlab tree pagination repeated page "2"`) + require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page") +} + +// TestPaginationHopCapStopsWalk drives each walk with a server that always +// advertises a fresh page number, so only the hop cap can end it. +func TestPaginationHopCapStopsWalk(t *testing.T) { + for _, tc := range []struct { + name string + want string + walk func(context.Context, *client) error + }{ + { + name: "projects", + want: "gitlab project pagination exceeded 10000 pages", + walk: func(ctx context.Context, c *client) error { + _, err := c.projects(ctx) + return err + }, + }, + { + name: "tree", + want: "gitlab tree pagination exceeded 10000 pages", + walk: func(ctx context.Context, c *client) error { + _, err := c.tree(ctx, "1", "main", "") + return err + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + allowLocalGitLabServer(t) + var requests int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt64(&requests, 1) + if n > maxPaginationHops+10 { + http.Error(w, "hop cap missing: too many requests", http.StatusTooManyRequests) + return + } + page, convErr := strconv.Atoi(r.URL.Query().Get("page")) + if convErr != nil { + http.Error(w, "unexpected page", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Next-Page", strconv.Itoa(page+1)) + _, _ = w.Write([]byte(`[]`)) + })) + t.Cleanup(server.Close) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + + var logs bytes.Buffer + logger.SetOutput(&logs) + t.Cleanup(logger.ConfigureFromEnv) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err = tc.walk(ctx, c) + + require.EqualError(t, err, tc.want) + require.Equal(t, int64(maxPaginationHops), atomic.LoadInt64(&requests)) + require.Contains(t, logs.String(), "pagination exceeded 10000 pages", + "hitting the cap must leave a warning in the log") + }) + } +} + +func TestTrackPageAllowsForwardPages(t *testing.T) { + seen := make(map[string]struct{}) + for _, page := range []string{"1", "2", "3"} { + require.NoError(t, trackPage(context.Background(), "project", seen, page)) + } + require.Error(t, trackPage(context.Background(), "project", seen, "2")) +}