fix(confluence): confirm unlisted pages before deleting after a cut-short listing
On a Server/DC instance with the #3596 behaviour every listing ends early, so skipping deletion reconciliation for such runs meant pages deleted in Confluence were never removed from the knowledge base. When a listing is cut short, look up each previously synced page it did not return with GET /rest/api/content/{id}?expand=space and delete it only when Confluence confirms it is gone from the space: a 404, a non-current status, or a different space. Pages that are still there, that fail to load (including 403), or that exceed the 200-probe budget per run stay tracked and are checked again next time.main
parent
9f57bbe4da
commit
503f1abfb4
|
|
@ -3,6 +3,7 @@ package confluence
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
|
@ -26,6 +27,9 @@ const (
|
|||
// next past the end of a space (#3596); permission filtering can also
|
||||
// empty one page mid-listing, so a single empty page is not the end.
|
||||
maxEmptyServerPages = 3
|
||||
// maxDeletionProbes caps how many unlisted pages one sync looks up
|
||||
// individually after a cut-short listing; the rest wait for the next run.
|
||||
maxDeletionProbes = 200
|
||||
)
|
||||
|
||||
type apiError struct {
|
||||
|
|
@ -367,6 +371,31 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo
|
|||
return all, complete, err
|
||||
}
|
||||
|
||||
// pageInSpace reports whether a Server page is still a current page of the
|
||||
// space. Confluence answers 404 for a page that was deleted or moved to the
|
||||
// trash; a page that is no longer current or now belongs to another space is
|
||||
// gone from this one too, matching what a complete listing would omit.
|
||||
func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {
|
||||
var result struct {
|
||||
Status string `json:"status"`
|
||||
Space struct {
|
||||
Key string `json:"key"`
|
||||
} `json:"space"`
|
||||
}
|
||||
err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=space", &result)
|
||||
var apiErr *apiError
|
||||
if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if result.Status != "" && result.Status != "current" {
|
||||
return false, nil
|
||||
}
|
||||
return result.Space.Key == "" || result.Space.Key == spaceKey, nil
|
||||
}
|
||||
|
||||
func (c *client) body(ctx context.Context, id string) (pageBody, error) {
|
||||
if c.cfg.cloud() {
|
||||
var result pageBody
|
||||
|
|
|
|||
|
|
@ -177,3 +177,34 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
|
|||
t.Fatalf("depth query = %#v", depths)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageInSpace(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("expand") != "space" {
|
||||
t.Errorf("expand = %q, want space", r.URL.Query().Get("expand"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/rest/api/content/current":
|
||||
_, _ = w.Write([]byte(`{"id": "current", "status": "current", "space": {"key": "ENG"}}`))
|
||||
case "/rest/api/content/trashed":
|
||||
_, _ = w.Write([]byte(`{"id": "trashed", "status": "trashed", "space": {"key": "ENG"}}`))
|
||||
case "/rest/api/content/moved":
|
||||
_, _ = w.Write([]byte(`{"id": "moved", "status": "current", "space": {"key": "OPS"}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
|
||||
for id, want := range map[string]bool{"current": true, "trashed": false, "moved": false, "deleted": false} {
|
||||
got, err := c.pageInSpace(context.Background(), id, "ENG")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", id, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("pageInSpace(%s) = %v, want %v", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ func (c *Connector) fetchStream(
|
|||
if !complete {
|
||||
logger.Warnf(ctx,
|
||||
"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
|
||||
"imported %d pages and skipped deletion reconciliation",
|
||||
"listed %d pages, checking unlisted ones one by one before deleting",
|
||||
s.Key, maxEmptyServerPages, len(pages))
|
||||
}
|
||||
priorPages, hadBaseline := baseline.SpacePages[resourceID]
|
||||
|
|
@ -229,13 +229,8 @@ func (c *Connector) fetchStream(
|
|||
}
|
||||
}
|
||||
if !complete {
|
||||
// A cut-short listing proves nothing about the pages it did not
|
||||
// reach: keep tracking them so a later complete listing can still
|
||||
// reconcile them, and delete nothing now.
|
||||
for id, version := range priorPages {
|
||||
if _, exists := seen[id]; !exists {
|
||||
next.SpacePages[resourceID][id] = version
|
||||
}
|
||||
if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if hadBaseline {
|
||||
if len(pages) == 0 && len(priorPages) > 0 {
|
||||
|
|
@ -260,15 +255,7 @@ func (c *Connector) fetchStream(
|
|||
if _, exists := seen[id]; exists {
|
||||
continue
|
||||
}
|
||||
deleted := types.FetchedItem{
|
||||
ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{"channel": types.ChannelConfluence},
|
||||
}
|
||||
if err := h.Emit(ctx, deleted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delete(next.SpacePages[resourceID], id)
|
||||
if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {
|
||||
if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
|
@ -279,6 +266,63 @@ func (c *Connector) fetchStream(
|
|||
return next.syncCursor(), nil
|
||||
}
|
||||
|
||||
// reconcileUnlisted handles the pages a cut-short listing did not return. The
|
||||
// listing proves nothing about them, so each one is looked up on its own and
|
||||
// deleted only when Confluence confirms it is gone from the space. Pages that
|
||||
// are still there, could not be checked, or exceed the per-run probe budget
|
||||
// stay tracked with their prior version and are checked again next run.
|
||||
func reconcileUnlisted(
|
||||
ctx context.Context,
|
||||
c *client,
|
||||
h datasource.StreamHandler,
|
||||
s space,
|
||||
resourceID string,
|
||||
priorPages map[string]string,
|
||||
seen map[string]struct{},
|
||||
next *cursor,
|
||||
) error {
|
||||
probed := 0
|
||||
for id, version := range priorPages {
|
||||
if _, exists := seen[id]; exists {
|
||||
continue
|
||||
}
|
||||
if probed >= maxDeletionProbes {
|
||||
next.SpacePages[resourceID][id] = version
|
||||
continue
|
||||
}
|
||||
probed++
|
||||
present, err := c.pageInSpace(ctx, id, s.Key)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)
|
||||
next.SpacePages[resourceID][id] = version
|
||||
continue
|
||||
}
|
||||
if present {
|
||||
next.SpacePages[resourceID][id] = version
|
||||
continue
|
||||
}
|
||||
if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {
|
||||
deleted := types.FetchedItem{
|
||||
ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{"channel": types.ChannelConfluence},
|
||||
}
|
||||
if err := h.Emit(ctx, deleted); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(next.SpacePages[resourceID], id)
|
||||
return h.Checkpoint(ctx, next.syncCursor())
|
||||
}
|
||||
|
||||
func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {
|
||||
code, reason := classifyConfluenceError(err)
|
||||
return types.FetchedItem{
|
||||
|
|
|
|||
|
|
@ -34,6 +34,16 @@ type streamAPI struct {
|
|||
// endlessNext mimics Server/Data Center builds that keep linking to a
|
||||
// next page after the last one, returning empty results forever (#3596).
|
||||
endlessNext bool
|
||||
// probe answers the per-page existence check (GET content/{id}?expand=space)
|
||||
// for pages that are not in the listing: an HTTP status, or 200 with the
|
||||
// space key the page now belongs to.
|
||||
probe map[string]probeReply
|
||||
probeCalls int
|
||||
}
|
||||
|
||||
type probeReply struct {
|
||||
status int
|
||||
spaceKey string
|
||||
}
|
||||
|
||||
func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {
|
||||
|
|
@ -84,6 +94,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
|
|||
"results": results,
|
||||
"_links": map[string]any{"next": next},
|
||||
})
|
||||
case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space":
|
||||
a.probeCalls++
|
||||
id := strings.TrimPrefix(path, "/wiki/rest/api/content/")
|
||||
reply, ok := a.probe[id]
|
||||
if !ok {
|
||||
return nil, errors.New("unexpected existence check for page " + id)
|
||||
}
|
||||
if reply.status != http.StatusOK {
|
||||
return &http.Response{
|
||||
StatusCode: reply.status,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader("{}")),
|
||||
}, nil
|
||||
}
|
||||
return a.jsonResponse(map[string]any{
|
||||
"id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey},
|
||||
})
|
||||
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
|
||||
if req.URL.Query().Get("expand") != "body.view,version,space" {
|
||||
return nil, errors.New("page body did not request rendered body.view")
|
||||
|
|
@ -403,12 +430,23 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {
|
||||
func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {
|
||||
for _, full := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
|
||||
api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}
|
||||
api := &streamAPI{
|
||||
endlessNext: true,
|
||||
pages: []streamPage{{id: "p1", title: "Page", version: 2}},
|
||||
probe: map[string]probeReply{
|
||||
"gone": {status: http.StatusNotFound},
|
||||
"moved": {status: http.StatusOK, spaceKey: "OPS"},
|
||||
"kept": {status: http.StatusOK, spaceKey: "ENG"},
|
||||
"denied": {status: http.StatusForbidden},
|
||||
},
|
||||
}
|
||||
h := &captureHandler{}
|
||||
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
|
||||
old := streamCursor(map[string]string{
|
||||
"p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1",
|
||||
})
|
||||
connector := newStreamConnector(api)
|
||||
fetch := connector.FetchStream
|
||||
if full {
|
||||
|
|
@ -421,17 +459,27 @@ func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T)
|
|||
if api.listCalls != 1+maxEmptyServerPages {
|
||||
t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
|
||||
}
|
||||
if api.probeCalls != 4 {
|
||||
t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls)
|
||||
}
|
||||
deleted := map[string]bool{}
|
||||
for _, item := range h.items {
|
||||
if item.IsDeleted {
|
||||
t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)
|
||||
deleted[item.ExternalID] = true
|
||||
}
|
||||
}
|
||||
if len(h.items) != 1 || h.items[0].ExternalID != "p1" {
|
||||
t.Fatalf("items = %#v, want the changed page p1 only", h.items)
|
||||
if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] {
|
||||
t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted)
|
||||
}
|
||||
pages := decodeCursor(got).SpacePages["1"]
|
||||
if pages["p1"] != "v:2" || pages["p2"] != "v:1" {
|
||||
t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)
|
||||
want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"}
|
||||
if len(pages) != len(want) {
|
||||
t.Fatalf("cursor pages = %#v, want %#v", pages, want)
|
||||
}
|
||||
for id, version := range want {
|
||||
if pages[id] != version {
|
||||
t.Fatalf("cursor pages = %#v, want %#v", pages, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue