fix(datasource): fail closed on GitLab compare errors

main
hahahahahayesyeseys 2026-09-23 08:04:54 +00:00 committed by lyingbug
parent 1fa915d712
commit 11edd16819
2 changed files with 178 additions and 36 deletions

View File

@ -178,39 +178,26 @@ func (c *Connector) FetchIncremental(ctx context.Context, ds *types.DataSourceCo
out = append(out, item)
}
} else if previous != head {
diff, err := c.client.compare(ctx, s.ProjectID, previous, head)
if err != nil || diff.CompareTimeout {
// A compare may be unavailable after history rewrites or truncated by
// the server. Re-enumerate the configured scope to preserve updates.
files, listErr := c.files(ctx, s.ProjectID, ref, s.Paths)
if listErr != nil {
return nil, nil, fmt.Errorf("gitlab list files %s: %w", s.ProjectID, listErr)
}
for _, f := range files {
item, itemErr := c.item(ctx, p, ref, f)
if itemErr != nil {
return nil, nil, itemErr
}
out = append(out, item)
}
} else {
for _, d := range diff.Diffs {
if d.DeletedFile {
if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
out = append(out, c.deleted(p, ref, d.OldPath))
}
continue
}
if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
diff, err := c.compareChanges(ctx, s.ProjectID, previous, head)
if err != nil {
return nil, nil, err
}
for _, d := range diff.Diffs {
if d.DeletedFile {
if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
out = append(out, c.deleted(p, ref, d.OldPath))
}
if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
item, err := c.item(ctx, p, ref, d.NewPath)
if err != nil {
return nil, nil, err
}
out = append(out, item)
continue
}
if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
out = append(out, c.deleted(p, ref, d.OldPath))
}
if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
item, err := c.item(ctx, p, ref, d.NewPath)
if err != nil {
return nil, nil, err
}
out = append(out, item)
}
}
}
@ -294,11 +281,9 @@ func gitLabCursor(value cursor) *types.SyncCursor {
func (c *Connector) streamChanges(
ctx context.Context, project *project, ref, from, to string, roots []string, h datasource.StreamHandler,
) error {
diff, err := c.client.compare(ctx, fmt.Sprint(project.ID), from, to)
if err != nil || diff.CompareTimeout {
// A compare can be unavailable after history rewrites or be truncated by
// GitLab. Re-enumerating the configured scope preserves file updates.
return c.streamFiles(ctx, project, ref, roots, h)
diff, err := c.compareChanges(ctx, fmt.Sprint(project.ID), from, to)
if err != nil {
return err
}
for _, change := range diff.Diffs {
if change.DeletedFile {
@ -327,6 +312,17 @@ func (c *Connector) streamChanges(
return nil
}
func (c *Connector) compareChanges(ctx context.Context, projectID, from, to string) (*comparison, error) {
diff, err := c.client.compare(ctx, projectID, from, to)
if err != nil {
return nil, fmt.Errorf("gitlab compare project %s: %w", projectID, err)
}
if diff.CompareTimeout {
return nil, fmt.Errorf("gitlab compare project %s timed out", projectID)
}
return diff, nil
}
func (c *Connector) streamFiles(
ctx context.Context, project *project, ref string, roots []string, h datasource.StreamHandler,
) error {

View File

@ -19,7 +19,7 @@ import (
// on the second page. Browsing remains shallow; fetching must visit every level.
func TestConnectorFetchesNestedDirectories(t *testing.T) {
for _, mode := range []string{
"all", "incremental", "stream", "incremental compare fallback", "stream compare fallback",
"all", "incremental", "stream",
} {
t.Run(mode, func(t *testing.T) {
allowLocalGitLabServer(t)
@ -107,3 +107,149 @@ func TestConnectorFetchesNestedDirectories(t *testing.T) {
})
}
}
func TestConnectorCompareFailuresDoNotFallBackToFullSync(t *testing.T) {
for _, failure := range []struct {
name string
statusCode int
body string
}{
{name: "timeout response", statusCode: http.StatusOK, body: `{"compare_timeout":true}`},
{name: "API error", statusCode: http.StatusInternalServerError, body: `{"message":"compare failed"}`},
} {
for _, mode := range []string{"incremental", "stream"} {
t.Run(failure.name+" "+mode, func(t *testing.T) {
allowLocalGitLabServer(t)
downstreamRequests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v4/projects/1":
_, _ = w.Write([]byte(`{
"id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
}`))
case "/api/v4/projects/1/repository/commits/main":
_, _ = w.Write([]byte(`{"id":"head"}`))
case "/api/v4/projects/1/repository/compare":
w.WriteHeader(failure.statusCode)
_, _ = w.Write([]byte(failure.body))
case "/api/v4/projects/1/repository/tree":
downstreamRequests++
_, _ = w.Write([]byte(`[{"name":"README.md","path":"README.md","type":"blob"}]`))
default:
if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
downstreamRequests++
_, _ = w.Write([]byte("# unchanged"))
return
}
http.NotFound(w, r)
}
}))
defer server.Close()
cfg := &types.DataSourceConfig{
Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
Settings: map[string]interface{}{"projects": []interface{}{
map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
}},
}
old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
connector := NewConnector()
switch mode {
case "incremental":
items, next, err := connector.FetchIncremental(context.Background(), cfg, old)
require.ErrorContains(t, err, "gitlab compare project 1")
require.Empty(t, items)
require.Nil(t, next)
case "stream":
handler := &gitLabStreamRecorder{}
next, err := connector.FetchStream(context.Background(), cfg, old, handler)
require.ErrorContains(t, err, "gitlab compare project 1")
require.Nil(t, next)
require.Empty(t, handler.items)
require.Empty(t, handler.checkpoints)
}
require.Zero(t, downstreamRequests, "compare failures must not trigger a full repository walk")
})
}
}
}
func TestConnectorSuccessfulCompareStillEmitsOnlyChangedFiles(t *testing.T) {
for _, mode := range []string{"incremental", "stream"} {
t.Run(mode, func(t *testing.T) {
allowLocalGitLabServer(t)
var compareQuery string
treeRequests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v4/projects/1":
_, _ = w.Write([]byte(`{
"id":1,"name":"docs","path_with_namespace":"group/docs",
"web_url":"https://gitlab.test/group/docs","default_branch":"main"
}`))
case "/api/v4/projects/1/repository/commits/main":
_, _ = w.Write([]byte(`{"id":"head"}`))
case "/api/v4/projects/1/repository/compare":
compareQuery = r.URL.RawQuery
_, _ = w.Write([]byte(`{"diffs":[
{"old_path":"removed.md","new_path":"removed.md","deleted_file":true},
{"old_path":"old.md","new_path":"renamed.md","renamed_file":true},
{"old_path":"guide.md","new_path":"guide.md"},
{"old_path":"server.go","new_path":"server.go"}
]}`))
case "/api/v4/projects/1/repository/tree":
treeRequests++
http.Error(w, "unexpected full repository walk", http.StatusInternalServerError)
default:
if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
_, _ = w.Write([]byte("# changed"))
return
}
http.NotFound(w, r)
}
}))
defer server.Close()
cfg := &types.DataSourceConfig{
Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
Settings: map[string]interface{}{"projects": []interface{}{
map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
}},
}
old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
connector := NewConnector()
var items []types.FetchedItem
var next *types.SyncCursor
switch mode {
case "incremental":
var err error
items, next, err = connector.FetchIncremental(context.Background(), cfg, old)
require.NoError(t, err)
case "stream":
handler := &gitLabStreamRecorder{}
var err error
next, err = connector.FetchStream(context.Background(), cfg, old, handler)
require.NoError(t, err)
items = handler.items
require.Len(t, handler.checkpoints, 1)
}
require.Equal(t, "from=previous&to=head", compareQuery)
require.Zero(t, treeRequests)
require.Len(t, items, 4)
require.True(t, items[0].IsDeleted)
require.Equal(t, "removed.md", items[0].Metadata["gitlab_path"])
require.True(t, items[1].IsDeleted)
require.Equal(t, "old.md", items[1].Metadata["gitlab_path"])
require.False(t, items[2].IsDeleted)
require.Equal(t, "renamed.md", items[2].Metadata["gitlab_path"])
require.False(t, items[3].IsDeleted)
require.Equal(t, "guide.md", items[3].Metadata["gitlab_path"])
require.Equal(t, "head", next.ConnectorCursor["projects"].(map[string]string)["1"])
})
}
}