diff --git a/internal/datasource/connector/gitlab/client.go b/internal/datasource/connector/gitlab/client.go index 5a26de62..3ebefb94 100644 --- a/internal/datasource/connector/gitlab/client.go +++ b/internal/datasource/connector/gitlab/client.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" ) const ( @@ -39,6 +40,11 @@ func base64FileBytes(rawLimit int64) int64 { return rawLimit/3*4 + (1 << 20) } +// maxPaginationHops bounds a single paginated walk. GitLab reports the next +// page through the X-Next-Page header, so a broken instance or reverse proxy +// could keep advertising pages indefinitely. +const maxPaginationHops = 10000 + type client struct { baseURL, token string http *http.Client @@ -206,13 +212,33 @@ func (c *client) project(ctx context.Context, id string) (*project, error) { return &p, err } +// trackPage records the page a paginated walk is about to request. A page that +// was already requested means X-Next-Page is not advancing, so the walk stops +// with an error instead of re-reading the same page; the hop cap bounds a +// server that keeps advertising fresh page numbers. +func trackPage(ctx context.Context, scope string, seen map[string]struct{}, page string) error { + if _, dup := seen[page]; dup { + return fmt.Errorf("gitlab %s pagination repeated page %q", scope, page) + } + if len(seen) >= maxPaginationHops { + logger.Warnf(ctx, "[GitLab] %s pagination exceeded %d pages; aborting", scope, maxPaginationHops) + return fmt.Errorf("gitlab %s pagination exceeded %d pages", scope, maxPaginationHops) + } + seen[page] = struct{}{} + return nil +} + func (c *client) projects(ctx context.Context) ([]project, error) { q := url.Values{ "membership": {"true"}, "per_page": {"100"}, "page": {"1"}, "order_by": {"path_with_namespace"}, "sort": {"asc"}, } var all []project + seen := make(map[string]struct{}) for { + if err := trackPage(ctx, "project", seen, q.Get("page")); err != nil { + return nil, err + } var page []project nextPage, err := c.getPage(ctx, "/projects", q, &page) if err != nil { @@ -251,7 +277,11 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er } endpoint := "/projects/" + projectPath(id) + "/repository/tree" var all []treeEntry + seen := make(map[string]struct{}) for { + if err := trackPage(ctx, "tree", seen, q.Get("page")); err != nil { + return nil, err + } var page []treeEntry nextPage, err := c.getPage(ctx, endpoint, q, &page) if err != nil { diff --git a/internal/datasource/connector/gitlab/pagination_guard_test.go b/internal/datasource/connector/gitlab/pagination_guard_test.go new file mode 100644 index 00000000..651e9571 --- /dev/null +++ b/internal/datasource/connector/gitlab/pagination_guard_test.go @@ -0,0 +1,146 @@ +package gitlab + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/stretchr/testify/require" +) + +// guardTestTimeout bounds every walk here so that a missing guard fails the +// test instead of hanging it. +const guardTestTimeout = 5 * time.Second + +// repeatedPageServer always advertises the same X-Next-Page value and refuses +// to serve more than three requests: a walk without a repeated-page guard then +// fails quickly with an unexpected error instead of spinning until the context +// deadline. +func repeatedPageServer(t *testing.T) (*httptest.Server, *int64) { + t.Helper() + allowLocalGitLabServer(t) + var requests int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt64(&requests, 1) > 3 { + http.Error(w, "pagination guard missing: too many requests", http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Next-Page", "2") + if strings.HasSuffix(r.URL.Path, "/repository/tree") { + _, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`)) + return + } + _, _ = w.Write([]byte(`[{"id":1,"path_with_namespace":"group/project-001"}]`)) + })) + t.Cleanup(server.Close) + return server, &requests +} + +func TestProjectsStopsOnRepeatedNextPage(t *testing.T) { + server, requests := repeatedPageServer(t) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout) + defer cancel() + + projects, err := c.projects(ctx) + + require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list") + require.Nil(t, projects) + require.Contains(t, err.Error(), `gitlab project pagination repeated page "2"`) + require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page") +} + +func TestTreeStopsOnRepeatedNextPage(t *testing.T) { + server, requests := repeatedPageServer(t) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout) + defer cancel() + + entries, err := c.tree(ctx, "1", "main", "") + + require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list") + require.Nil(t, entries) + require.Contains(t, err.Error(), `gitlab tree pagination repeated page "2"`) + require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page") +} + +// TestPaginationHopCapStopsWalk drives each walk with a server that always +// advertises a fresh page number, so only the hop cap can end it. +func TestPaginationHopCapStopsWalk(t *testing.T) { + for _, tc := range []struct { + name string + want string + walk func(context.Context, *client) error + }{ + { + name: "projects", + want: "gitlab project pagination exceeded 10000 pages", + walk: func(ctx context.Context, c *client) error { + _, err := c.projects(ctx) + return err + }, + }, + { + name: "tree", + want: "gitlab tree pagination exceeded 10000 pages", + walk: func(ctx context.Context, c *client) error { + _, err := c.tree(ctx, "1", "main", "") + return err + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + allowLocalGitLabServer(t) + var requests int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt64(&requests, 1) + if n > maxPaginationHops+10 { + http.Error(w, "hop cap missing: too many requests", http.StatusTooManyRequests) + return + } + page, convErr := strconv.Atoi(r.URL.Query().Get("page")) + if convErr != nil { + http.Error(w, "unexpected page", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Next-Page", strconv.Itoa(page+1)) + _, _ = w.Write([]byte(`[]`)) + })) + t.Cleanup(server.Close) + c, err := newClient(server.URL, "test-token") + require.NoError(t, err) + + var logs bytes.Buffer + logger.SetOutput(&logs) + t.Cleanup(logger.ConfigureFromEnv) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err = tc.walk(ctx, c) + + require.EqualError(t, err, tc.want) + require.Equal(t, int64(maxPaginationHops), atomic.LoadInt64(&requests)) + require.Contains(t, logs.String(), "pagination exceeded 10000 pages", + "hitting the cap must leave a warning in the log") + }) + } +} + +func TestTrackPageAllowsForwardPages(t *testing.T) { + seen := make(map[string]struct{}) + for _, page := range []string{"1", "2", "3"} { + require.NoError(t, trackPage(context.Background(), "project", seen, page)) + } + require.Error(t, trackPage(context.Background(), "project", seen, "2")) +}