diff --git a/internal/datasource/connector/gitlab/connector.go b/internal/datasource/connector/gitlab/connector.go index b8241954..0a71e7a9 100644 --- a/internal/datasource/connector/gitlab/connector.go +++ b/internal/datasource/connector/gitlab/connector.go @@ -178,39 +178,26 @@ func (c *Connector) FetchIncremental(ctx context.Context, ds *types.DataSourceCo out = append(out, item) } } else if previous != head { - diff, err := c.client.compare(ctx, s.ProjectID, previous, head) - if err != nil || diff.CompareTimeout { - // A compare may be unavailable after history rewrites or truncated by - // the server. Re-enumerate the configured scope to preserve updates. - files, listErr := c.files(ctx, s.ProjectID, ref, s.Paths) - if listErr != nil { - return nil, nil, fmt.Errorf("gitlab list files %s: %w", s.ProjectID, listErr) - } - for _, f := range files { - item, itemErr := c.item(ctx, p, ref, f) - if itemErr != nil { - return nil, nil, itemErr - } - out = append(out, item) - } - } else { - for _, d := range diff.Diffs { - if d.DeletedFile { - if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) { - out = append(out, c.deleted(p, ref, d.OldPath)) - } - continue - } - if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) { + diff, err := c.compareChanges(ctx, s.ProjectID, previous, head) + if err != nil { + return nil, nil, err + } + for _, d := range diff.Diffs { + if d.DeletedFile { + if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) { out = append(out, c.deleted(p, ref, d.OldPath)) } - if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) { - item, err := c.item(ctx, p, ref, d.NewPath) - if err != nil { - return nil, nil, err - } - out = append(out, item) + continue + } + if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) { + out = append(out, c.deleted(p, ref, d.OldPath)) + } + if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) { + item, err := c.item(ctx, p, ref, d.NewPath) + if err != nil { + return nil, nil, err } + out = append(out, item) } } } @@ -294,11 +281,9 @@ func gitLabCursor(value cursor) *types.SyncCursor { func (c *Connector) streamChanges( ctx context.Context, project *project, ref, from, to string, roots []string, h datasource.StreamHandler, ) error { - diff, err := c.client.compare(ctx, fmt.Sprint(project.ID), from, to) - if err != nil || diff.CompareTimeout { - // A compare can be unavailable after history rewrites or be truncated by - // GitLab. Re-enumerating the configured scope preserves file updates. - return c.streamFiles(ctx, project, ref, roots, h) + diff, err := c.compareChanges(ctx, fmt.Sprint(project.ID), from, to) + if err != nil { + return err } for _, change := range diff.Diffs { if change.DeletedFile { @@ -327,6 +312,17 @@ func (c *Connector) streamChanges( return nil } +func (c *Connector) compareChanges(ctx context.Context, projectID, from, to string) (*comparison, error) { + diff, err := c.client.compare(ctx, projectID, from, to) + if err != nil { + return nil, fmt.Errorf("gitlab compare project %s: %w", projectID, err) + } + if diff.CompareTimeout { + return nil, fmt.Errorf("gitlab compare project %s timed out", projectID) + } + return diff, nil +} + func (c *Connector) streamFiles( ctx context.Context, project *project, ref string, roots []string, h datasource.StreamHandler, ) error { diff --git a/internal/datasource/connector/gitlab/connector_recursion_test.go b/internal/datasource/connector/gitlab/connector_recursion_test.go index f3d82bac..5025ffc7 100644 --- a/internal/datasource/connector/gitlab/connector_recursion_test.go +++ b/internal/datasource/connector/gitlab/connector_recursion_test.go @@ -19,7 +19,7 @@ import ( // on the second page. Browsing remains shallow; fetching must visit every level. func TestConnectorFetchesNestedDirectories(t *testing.T) { for _, mode := range []string{ - "all", "incremental", "stream", "incremental compare fallback", "stream compare fallback", + "all", "incremental", "stream", } { t.Run(mode, func(t *testing.T) { allowLocalGitLabServer(t) @@ -107,3 +107,149 @@ func TestConnectorFetchesNestedDirectories(t *testing.T) { }) } } + +func TestConnectorCompareFailuresDoNotFallBackToFullSync(t *testing.T) { + for _, failure := range []struct { + name string + statusCode int + body string + }{ + {name: "timeout response", statusCode: http.StatusOK, body: `{"compare_timeout":true}`}, + {name: "API error", statusCode: http.StatusInternalServerError, body: `{"message":"compare failed"}`}, + } { + for _, mode := range []string{"incremental", "stream"} { + t.Run(failure.name+" "+mode, func(t *testing.T) { + allowLocalGitLabServer(t) + downstreamRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v4/projects/1": + _, _ = w.Write([]byte(`{ + "id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main" + }`)) + case "/api/v4/projects/1/repository/commits/main": + _, _ = w.Write([]byte(`{"id":"head"}`)) + case "/api/v4/projects/1/repository/compare": + w.WriteHeader(failure.statusCode) + _, _ = w.Write([]byte(failure.body)) + case "/api/v4/projects/1/repository/tree": + downstreamRequests++ + _, _ = w.Write([]byte(`[{"name":"README.md","path":"README.md","type":"blob"}]`)) + default: + if strings.Contains(r.URL.EscapedPath(), "/repository/files/") { + downstreamRequests++ + _, _ = w.Write([]byte("# unchanged")) + return + } + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg := &types.DataSourceConfig{ + Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"}, + Settings: map[string]interface{}{"projects": []interface{}{ + map[string]interface{}{"project_id": "1", "paths": []interface{}{}}, + }}, + } + old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}}) + connector := NewConnector() + + switch mode { + case "incremental": + items, next, err := connector.FetchIncremental(context.Background(), cfg, old) + require.ErrorContains(t, err, "gitlab compare project 1") + require.Empty(t, items) + require.Nil(t, next) + case "stream": + handler := &gitLabStreamRecorder{} + next, err := connector.FetchStream(context.Background(), cfg, old, handler) + require.ErrorContains(t, err, "gitlab compare project 1") + require.Nil(t, next) + require.Empty(t, handler.items) + require.Empty(t, handler.checkpoints) + } + require.Zero(t, downstreamRequests, "compare failures must not trigger a full repository walk") + }) + } + } +} + +func TestConnectorSuccessfulCompareStillEmitsOnlyChangedFiles(t *testing.T) { + for _, mode := range []string{"incremental", "stream"} { + t.Run(mode, func(t *testing.T) { + allowLocalGitLabServer(t) + var compareQuery string + treeRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v4/projects/1": + _, _ = w.Write([]byte(`{ + "id":1,"name":"docs","path_with_namespace":"group/docs", + "web_url":"https://gitlab.test/group/docs","default_branch":"main" + }`)) + case "/api/v4/projects/1/repository/commits/main": + _, _ = w.Write([]byte(`{"id":"head"}`)) + case "/api/v4/projects/1/repository/compare": + compareQuery = r.URL.RawQuery + _, _ = w.Write([]byte(`{"diffs":[ + {"old_path":"removed.md","new_path":"removed.md","deleted_file":true}, + {"old_path":"old.md","new_path":"renamed.md","renamed_file":true}, + {"old_path":"guide.md","new_path":"guide.md"}, + {"old_path":"server.go","new_path":"server.go"} + ]}`)) + case "/api/v4/projects/1/repository/tree": + treeRequests++ + http.Error(w, "unexpected full repository walk", http.StatusInternalServerError) + default: + if strings.Contains(r.URL.EscapedPath(), "/repository/files/") { + _, _ = w.Write([]byte("# changed")) + return + } + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg := &types.DataSourceConfig{ + Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"}, + Settings: map[string]interface{}{"projects": []interface{}{ + map[string]interface{}{"project_id": "1", "paths": []interface{}{}}, + }}, + } + old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}}) + connector := NewConnector() + var items []types.FetchedItem + var next *types.SyncCursor + + switch mode { + case "incremental": + var err error + items, next, err = connector.FetchIncremental(context.Background(), cfg, old) + require.NoError(t, err) + case "stream": + handler := &gitLabStreamRecorder{} + var err error + next, err = connector.FetchStream(context.Background(), cfg, old, handler) + require.NoError(t, err) + items = handler.items + require.Len(t, handler.checkpoints, 1) + } + + require.Equal(t, "from=previous&to=head", compareQuery) + require.Zero(t, treeRequests) + require.Len(t, items, 4) + require.True(t, items[0].IsDeleted) + require.Equal(t, "removed.md", items[0].Metadata["gitlab_path"]) + require.True(t, items[1].IsDeleted) + require.Equal(t, "old.md", items[1].Metadata["gitlab_path"]) + require.False(t, items[2].IsDeleted) + require.Equal(t, "renamed.md", items[2].Metadata["gitlab_path"]) + require.False(t, items[3].IsDeleted) + require.Equal(t, "guide.md", items[3].Metadata["gitlab_path"]) + require.Equal(t, "head", next.ConnectorCursor["projects"].(map[string]string)["1"]) + }) + } +}