fix(datasource): fail closed on GitLab compare errors
parent
1fa915d712
commit
11edd16819
|
|
@ -178,39 +178,26 @@ func (c *Connector) FetchIncremental(ctx context.Context, ds *types.DataSourceCo
|
|||
out = append(out, item)
|
||||
}
|
||||
} else if previous != head {
|
||||
diff, err := c.client.compare(ctx, s.ProjectID, previous, head)
|
||||
if err != nil || diff.CompareTimeout {
|
||||
// A compare may be unavailable after history rewrites or truncated by
|
||||
// the server. Re-enumerate the configured scope to preserve updates.
|
||||
files, listErr := c.files(ctx, s.ProjectID, ref, s.Paths)
|
||||
if listErr != nil {
|
||||
return nil, nil, fmt.Errorf("gitlab list files %s: %w", s.ProjectID, listErr)
|
||||
}
|
||||
for _, f := range files {
|
||||
item, itemErr := c.item(ctx, p, ref, f)
|
||||
if itemErr != nil {
|
||||
return nil, nil, itemErr
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
} else {
|
||||
for _, d := range diff.Diffs {
|
||||
if d.DeletedFile {
|
||||
if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
|
||||
out = append(out, c.deleted(p, ref, d.OldPath))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
|
||||
diff, err := c.compareChanges(ctx, s.ProjectID, previous, head)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, d := range diff.Diffs {
|
||||
if d.DeletedFile {
|
||||
if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
|
||||
out = append(out, c.deleted(p, ref, d.OldPath))
|
||||
}
|
||||
if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
|
||||
item, err := c.item(ctx, p, ref, d.NewPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
continue
|
||||
}
|
||||
if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
|
||||
out = append(out, c.deleted(p, ref, d.OldPath))
|
||||
}
|
||||
if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
|
||||
item, err := c.item(ctx, p, ref, d.NewPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -294,11 +281,9 @@ func gitLabCursor(value cursor) *types.SyncCursor {
|
|||
func (c *Connector) streamChanges(
|
||||
ctx context.Context, project *project, ref, from, to string, roots []string, h datasource.StreamHandler,
|
||||
) error {
|
||||
diff, err := c.client.compare(ctx, fmt.Sprint(project.ID), from, to)
|
||||
if err != nil || diff.CompareTimeout {
|
||||
// A compare can be unavailable after history rewrites or be truncated by
|
||||
// GitLab. Re-enumerating the configured scope preserves file updates.
|
||||
return c.streamFiles(ctx, project, ref, roots, h)
|
||||
diff, err := c.compareChanges(ctx, fmt.Sprint(project.ID), from, to)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, change := range diff.Diffs {
|
||||
if change.DeletedFile {
|
||||
|
|
@ -327,6 +312,17 @@ func (c *Connector) streamChanges(
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Connector) compareChanges(ctx context.Context, projectID, from, to string) (*comparison, error) {
|
||||
diff, err := c.client.compare(ctx, projectID, from, to)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gitlab compare project %s: %w", projectID, err)
|
||||
}
|
||||
if diff.CompareTimeout {
|
||||
return nil, fmt.Errorf("gitlab compare project %s timed out", projectID)
|
||||
}
|
||||
return diff, nil
|
||||
}
|
||||
|
||||
func (c *Connector) streamFiles(
|
||||
ctx context.Context, project *project, ref string, roots []string, h datasource.StreamHandler,
|
||||
) error {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import (
|
|||
// on the second page. Browsing remains shallow; fetching must visit every level.
|
||||
func TestConnectorFetchesNestedDirectories(t *testing.T) {
|
||||
for _, mode := range []string{
|
||||
"all", "incremental", "stream", "incremental compare fallback", "stream compare fallback",
|
||||
"all", "incremental", "stream",
|
||||
} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
allowLocalGitLabServer(t)
|
||||
|
|
@ -107,3 +107,149 @@ func TestConnectorFetchesNestedDirectories(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorCompareFailuresDoNotFallBackToFullSync(t *testing.T) {
|
||||
for _, failure := range []struct {
|
||||
name string
|
||||
statusCode int
|
||||
body string
|
||||
}{
|
||||
{name: "timeout response", statusCode: http.StatusOK, body: `{"compare_timeout":true}`},
|
||||
{name: "API error", statusCode: http.StatusInternalServerError, body: `{"message":"compare failed"}`},
|
||||
} {
|
||||
for _, mode := range []string{"incremental", "stream"} {
|
||||
t.Run(failure.name+" "+mode, func(t *testing.T) {
|
||||
allowLocalGitLabServer(t)
|
||||
downstreamRequests := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/v4/projects/1":
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
|
||||
}`))
|
||||
case "/api/v4/projects/1/repository/commits/main":
|
||||
_, _ = w.Write([]byte(`{"id":"head"}`))
|
||||
case "/api/v4/projects/1/repository/compare":
|
||||
w.WriteHeader(failure.statusCode)
|
||||
_, _ = w.Write([]byte(failure.body))
|
||||
case "/api/v4/projects/1/repository/tree":
|
||||
downstreamRequests++
|
||||
_, _ = w.Write([]byte(`[{"name":"README.md","path":"README.md","type":"blob"}]`))
|
||||
default:
|
||||
if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
|
||||
downstreamRequests++
|
||||
_, _ = w.Write([]byte("# unchanged"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
|
||||
Settings: map[string]interface{}{"projects": []interface{}{
|
||||
map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
|
||||
}},
|
||||
}
|
||||
old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
|
||||
connector := NewConnector()
|
||||
|
||||
switch mode {
|
||||
case "incremental":
|
||||
items, next, err := connector.FetchIncremental(context.Background(), cfg, old)
|
||||
require.ErrorContains(t, err, "gitlab compare project 1")
|
||||
require.Empty(t, items)
|
||||
require.Nil(t, next)
|
||||
case "stream":
|
||||
handler := &gitLabStreamRecorder{}
|
||||
next, err := connector.FetchStream(context.Background(), cfg, old, handler)
|
||||
require.ErrorContains(t, err, "gitlab compare project 1")
|
||||
require.Nil(t, next)
|
||||
require.Empty(t, handler.items)
|
||||
require.Empty(t, handler.checkpoints)
|
||||
}
|
||||
require.Zero(t, downstreamRequests, "compare failures must not trigger a full repository walk")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorSuccessfulCompareStillEmitsOnlyChangedFiles(t *testing.T) {
|
||||
for _, mode := range []string{"incremental", "stream"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
allowLocalGitLabServer(t)
|
||||
var compareQuery string
|
||||
treeRequests := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/v4/projects/1":
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id":1,"name":"docs","path_with_namespace":"group/docs",
|
||||
"web_url":"https://gitlab.test/group/docs","default_branch":"main"
|
||||
}`))
|
||||
case "/api/v4/projects/1/repository/commits/main":
|
||||
_, _ = w.Write([]byte(`{"id":"head"}`))
|
||||
case "/api/v4/projects/1/repository/compare":
|
||||
compareQuery = r.URL.RawQuery
|
||||
_, _ = w.Write([]byte(`{"diffs":[
|
||||
{"old_path":"removed.md","new_path":"removed.md","deleted_file":true},
|
||||
{"old_path":"old.md","new_path":"renamed.md","renamed_file":true},
|
||||
{"old_path":"guide.md","new_path":"guide.md"},
|
||||
{"old_path":"server.go","new_path":"server.go"}
|
||||
]}`))
|
||||
case "/api/v4/projects/1/repository/tree":
|
||||
treeRequests++
|
||||
http.Error(w, "unexpected full repository walk", http.StatusInternalServerError)
|
||||
default:
|
||||
if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
|
||||
_, _ = w.Write([]byte("# changed"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
|
||||
Settings: map[string]interface{}{"projects": []interface{}{
|
||||
map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
|
||||
}},
|
||||
}
|
||||
old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
|
||||
connector := NewConnector()
|
||||
var items []types.FetchedItem
|
||||
var next *types.SyncCursor
|
||||
|
||||
switch mode {
|
||||
case "incremental":
|
||||
var err error
|
||||
items, next, err = connector.FetchIncremental(context.Background(), cfg, old)
|
||||
require.NoError(t, err)
|
||||
case "stream":
|
||||
handler := &gitLabStreamRecorder{}
|
||||
var err error
|
||||
next, err = connector.FetchStream(context.Background(), cfg, old, handler)
|
||||
require.NoError(t, err)
|
||||
items = handler.items
|
||||
require.Len(t, handler.checkpoints, 1)
|
||||
}
|
||||
|
||||
require.Equal(t, "from=previous&to=head", compareQuery)
|
||||
require.Zero(t, treeRequests)
|
||||
require.Len(t, items, 4)
|
||||
require.True(t, items[0].IsDeleted)
|
||||
require.Equal(t, "removed.md", items[0].Metadata["gitlab_path"])
|
||||
require.True(t, items[1].IsDeleted)
|
||||
require.Equal(t, "old.md", items[1].Metadata["gitlab_path"])
|
||||
require.False(t, items[2].IsDeleted)
|
||||
require.Equal(t, "renamed.md", items[2].Metadata["gitlab_path"])
|
||||
require.False(t, items[3].IsDeleted)
|
||||
require.Equal(t, "guide.md", items[3].Metadata["gitlab_path"])
|
||||
require.Equal(t, "head", next.ConnectorCursor["projects"].(map[string]string)["1"])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue