From fe444c851ef4430bcdcbef4cb10342839c57807f Mon Sep 17 00:00:00 2001 From: Lyang77 <563647186@qq.com> Date: Mon, 12 Oct 2026 13:00:33 +0800 Subject: [PATCH] fix(feishu): add shared request pacing and improve HTTP 429 / error 99991400 retries in Feishu Wiki synchronization (#4039) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(feishu): 修复同步请求限流与共享重试等待 * fix(feishu): 完善知识库限流冷却和导出轮询 Bound export polling by poll count so shared pacing and cooldowns retain the same export ticket without causing false 60-second timeouts. Preserve the feishu_timeout failure classification when the poll budget is exhausted. Cap cooldowns, isolate reactive waits by API path, and use a configurable 10-second default when no valid retry hint is available. Refresh expired tokens before rejoining the request gate to preserve pacing and new cooldowns. Skip JSON decoding when the legacy rate-limit code is absent, derive the prefilter from the shared protocol constant, and update configuration docs. Isolate shared gates in tests and cover cooldowns, cancellation, API grouping, concurrent token refresh, and timeout classification. Validation: Feishu core tests pass; diff-scoped Feishu lint reports 0 issues; gofmt and git diff --check pass. No live Feishu sync was run for this commit. * fix(feishu): grant request gate slots in arrival order and improve rate-limit logs - Reserve the next slot before sleeping so concurrent callers are paced in FIFO order instead of racing after a shared wake-up. - Restore the error log for non-rate-limited HTTP 400 download failures. - Include the API gate name in rate-limit warnings. --------- Co-authored-by: wizardchen --- .env.example | 12 + internal/datasource/README.md | 57 +- .../connector/feishu/core/client.go | 120 ++-- .../feishu/core/client_retry_test.go | 110 ++-- .../connector/feishu/core/request_limits.go | 238 ++++++++ .../feishu/core/request_limits_test.go | 530 ++++++++++++++++++ .../feishu/core/request_token_test.go | 121 ++++ .../connector/feishu/core/shared.go | 4 +- 8 files changed, 1098 insertions(+), 94 deletions(-) create mode 100644 internal/datasource/connector/feishu/core/request_limits.go create mode 100644 internal/datasource/connector/feishu/core/request_limits_test.go create mode 100644 internal/datasource/connector/feishu/core/request_token_test.go diff --git a/.env.example b/.env.example index 1248e3c3..59ae4ddf 100644 --- a/.env.example +++ b/.env.example @@ -551,6 +551,18 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # 此方案为临时方案,若后续有更好的解析方案,会移除此环境变量并替换。 # FEISHU_DOCX_PARSE_MODE=export +# 飞书导出创建 / 查询 / 下载、Wiki 子节点列表分别按应用共享配额(单进程)。 +# 默认每个接口 80 次/分钟,平滑发送、不积累突发额度;正整数可覆盖默认值。 +# FEISHU_API_REQUESTS_PER_MINUTE=80 +# 限流响应没有有效等待提示时的等待时长(Go duration,默认 10s)。 +# FEISHU_RATE_LIMIT_RETRY_WAIT=10s +# 每次共享冷却的最大等待时长,包含响应头提示和上述默认等待(Go duration)。 +# FEISHU_RATE_LIMIT_MAX_WAIT=5m +# 导出任务最多轮询次数;限流重试及排队不消耗轮询次数,任务上下文仍可取消。 +# FEISHU_EXPORT_MAX_POLLS=30 +# 导出任务两次轮询之间的间隔(Go duration)。 +# FEISHU_EXPORT_POLL_INTERVAL=2s + # ##################################################################### # F. 认证与空间隔离 # ##################################################################### diff --git a/internal/datasource/README.md b/internal/datasource/README.md index 9512515d..0b9b2bcb 100644 --- a/internal/datasource/README.md +++ b/internal/datasource/README.md @@ -234,11 +234,66 @@ GET /api/v1/datasource/types # Available connectors 5. **Multi-tenant**: All operations are tenant-isolated 6. **Channel Tracking**: Knowledge.Channel field tracks data source type for origin tracking +## Feishu request pacing and rate-limit retries + +Feishu export creation, export status polling, export downloads, and Wiki node +listing each use a separate request budget. Clients using the same API host and +App ID share these budgets within one backend process, including concurrent data +sources and task retries. Document tokens, export tickets, and query parameters +do not create additional budgets. + +| Environment variable | Default | Behavior | +| --- | --- | --- | +| `FEISHU_API_REQUESTS_PER_MINUTE` | `80` | Positive integer; requests to each paced API are evenly spaced (750 ms by default), without accumulating burst capacity. | +| `FEISHU_RATE_LIMIT_RETRY_WAIT` | `10s` | Positive Go duration used when the response supplies no valid reset/retry hint. | +| `FEISHU_RATE_LIMIT_MAX_WAIT` | `5m` | Positive Go duration capping each cooldown, including response-header hints and the fallback. | +| `FEISHU_EXPORT_MAX_POLLS` | `30` | Positive integer bounding status polls of the same export task; API retries do not consume additional polls. | +| `FEISHU_EXPORT_POLL_INTERVAL` | `2s` | Positive Go duration between status polls while the export is still processing. | + +Invalid values use the defaults. Other Feishu APIs retain their existing request +frequency, with reactive cooldowns scoped to the HTTP method and normalized API +path template. Document tokens, table IDs, sheet ranges, and query parameters do +not split a known API's cooldown. Media downloads, file downloads, docx blocks, +raw content, bitable fields/record searches, sheet values, folder metadata, Wiki +space listing, and Wiki node lookup use independent cooldowns. Unknown endpoints +use their service/version/resource prefix. Check the application's actual API +quotas before increasing the configured budget. + +JSON requests retry HTTP 429 and HTTP 400/200 responses containing Feishu error +code `99991400`. Downloads retry HTTP 429 and HTTP 400 with that error code. +The wait is the longer valid value from `x-ogw-ratelimit-reset` and `Retry-After`; +`Retry-After` also accepts an HTTP date. All hints are clamped to the configured +maximum so an unreasonable gateway header cannot indefinitely block later syncs. +Without a valid hint, a rate-limit response imposes a 10-second cooldown by +default. The three retries add a nominal 30 seconds of cooldown; shared pacing +or further rate-limit responses from other clients can extend the total wait. +This replaces the original 2/4/8-second backoff with a shared cooldown scoped +to the affected API. +Only callers of the same API share that cooldown. Requests retain the existing +limit of three retries. Even when retries are exhausted, subsequent documents and other +clients sharing that API wait for the cooldown. Waiting honors task cancellation. +Ordinary permission/validation errors still fail immediately, and download error +bodies retain their existing size bounds. + +Export polling uses a bounded number of polls rather than a 60-second wall-clock +deadline. Shared pacing and cooldown waits therefore do not cause false export +timeouts or create replacement tasks within the operation. With the defaults, +an export that remains processing is checked up to 30 times, with 2 seconds +between polls. The caller's context bounds total elapsed time, including waits +and retries. Exhausting the poll budget retains the `feishu_timeout` failure +classification. Tokens are checked again after gate waits; requests needing a +token refresh rejoin the gate afterwards, so both JSON requests and downloads +remain paced and honor any cooldown imposed during the refresh. + +These budgets are process-local. Multiple backend instances or other applications +using the same Feishu credentials may still hit the upstream quota; configure +their combined traffic accordingly. + ## Future Enhancements 1. **Webhook Support**: Real-time push from platforms that support webhooks 2. **Conflict Resolution**: Advanced merge/conflict strategies for overlapping content -3. **Rate Limiting**: Per-connector rate limiting and backoff strategies +3. **Rate Limiting**: Extend request pacing to other connectors and coordinate quotas across backend instances 4. **Scheduling**: Full cron scheduler with time zone support 5. **Monitoring**: Metrics, alerting, and sync health dashboards 6. **Filtering**: User-defined filters for selective syncing (by title, date, tags, etc) diff --git a/internal/datasource/connector/feishu/core/client.go b/internal/datasource/connector/feishu/core/client.go index 3fb3dbcf..d8eef921 100644 --- a/internal/datasource/connector/feishu/core/client.go +++ b/internal/datasource/connector/feishu/core/client.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "sync" "time" @@ -166,8 +165,33 @@ func (c *Client) GetTenantAccessToken(ctx context.Context) (string, error) { return c.tokenCache, nil } +// waitForRequestToken authenticates before pacing and rechecks the cached token +// afterwards. A refresh must happen before rejoining the gate, otherwise requests +// that have already been paced can pile up behind tokenMu and then burst together. +func (c *Client) waitForRequestToken(ctx context.Context, gate *feishuRequestGate) (string, error) { + for { + token, err := c.GetTenantAccessToken(ctx) + if err != nil { + return "", err + } + if err := gate.wait(ctx); err != nil { + return "", err + } + // Never wait on a token refresh after the gate has granted a request slot. + if !c.tokenMu.TryLock() { + continue + } + valid := token == c.tokenCache && time.Now().Before(c.tokenExpAt) + c.tokenMu.Unlock() + if valid { + return token, nil + } + } +} + // Retry policy shared by DoRequest (JSON API calls) and downloadRawBytes (file -// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off. +// downloads): rate limits honour Feishu's reset header / Retry-After, 5xx +// retries once, transport errors back off. const ( feishuMaxRetries = 3 feishuMax5xxRetries = 1 @@ -188,8 +212,8 @@ var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * t // retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive // export/wiki APIs are aggressively rate limited, and a thousand-document sync // issues tens of thousands of calls; without backoff a single 429 burst used to -// fail whole swathes of documents silently. 429 responses honour Retry-After; -// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx). +// fail whole swathes of documents silently. HTTP 400 with code 99991400 is +// also a rate limit. Ordinary permission/validation errors still fail fast. func (c *Client) DoRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error { const ( maxRetries = feishuMaxRetries @@ -198,7 +222,7 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf ) backoff := feishuRetryBackoff - token, err := c.GetTenantAccessToken(ctx) + _, err := c.GetTenantAccessToken(ctx) if err != nil { return err } @@ -213,8 +237,13 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf url := c.baseURL + path var lastErr error + gate := c.requestGate(method, path) for attempt := 0; attempt <= maxRetries; attempt++ { + token, err := c.waitForRequestToken(ctx, gate) + if err != nil { + return err + } var bodyReader io.Reader if bodyBytes != nil { bodyReader = bytes.NewReader(bodyBytes) @@ -260,13 +289,14 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s", method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000)) - if resp.StatusCode == http.StatusTooManyRequests { - wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)]) - lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500)) + if isFeishuRateLimited(resp.StatusCode, respBody) { + wait := feishuRateLimitWait(resp.Header, time.Now()) + gate.cooldown(wait) + logger.Warnf(ctx, "[Feishu] rate limited: api=%q wait=%s attempt=%d/%d", + gate.api, wait, attempt, maxRetries) + lastErr = fmt.Errorf("feishu rate limited: status=%d body=%s", + resp.StatusCode, truncate(string(respBody), 500)) if attempt < maxRetries { - if sErr := sleepCtx(ctx, wait); sErr != nil { - return sErr - } continue } return lastErr @@ -298,23 +328,6 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf return lastErr } -// parseRetryAfter interprets a Retry-After header value (seconds) into a wait -// duration, coercing 0/negative to a short delay and falling back when absent -// or unparseable. -func parseRetryAfter(header string, fallback time.Duration) time.Duration { - if header == "" { - return fallback - } - secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64) - if err != nil { - return fallback - } - if secs <= 0 { - return 100 * time.Millisecond - } - return time.Duration(secs * float64(time.Second)) -} - // sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the // context ends first so retries abort promptly on task cancellation/timeout. func sleepCtx(ctx context.Context, d time.Duration) error { @@ -646,10 +659,15 @@ func (c *Client) downloadExportFile(ctx context.Context, fileToken string) ([]by return c.downloadRawBytes(ctx, path) } +// errExportPollsExhausted keeps the error classification independent of the +// message's poll count and ticket details, including when connectors wrap it. +var errExportPollsExhausted = errors.New("export task still processing") + // ExportAndDownload is a high-level helper that creates an export task, polls until // completion, and downloads the resulting file. Returns (fileBytes, fileName, error). // -// Timeout: 60 seconds. Poll interval: 2 seconds. +// Poll count and interval are configurable. Pacing/cooldown waits do not consume +// the poll budget; the caller's context still bounds the entire operation. func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) { // Determine export format fileExt, ok := ObjTypeToExportFileExtension[objType] @@ -668,11 +686,13 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string return nil, "", err } - // Step 2: poll until ready (max 60s, every 2s) - deadline := time.Now().Add(60 * time.Second) + // Step 2: retain the same ticket while waiting on shared API limits. A wall + // clock deadline here would turn other documents' cooldowns into false failures. + maxPolls := feishuExportMaxPolls() + pollInterval := feishuConfiguredDuration("FEISHU_EXPORT_POLL_INTERVAL", defaultFeishuExportPollInterval) var fileToken, fileName string - for time.Now().Before(deadline) { + for poll := 0; poll < maxPolls; poll++ { fileToken, fileName, err = c.getExportTaskStatus(ctx, ticket, objToken) if err != nil { return nil, "", err @@ -680,15 +700,15 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string if fileToken != "" { break // export ready } - select { - case <-ctx.Done(): - return nil, "", ctx.Err() - case <-time.After(2 * time.Second): + if poll+1 < maxPolls { + if err := sleepCtx(ctx, pollInterval); err != nil { + return nil, "", err + } } } if fileToken == "" { - return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket) + return nil, "", fmt.Errorf("%w after %d polls (ticket=%s)", errExportPollsExhausted, maxPolls, ticket) } // Step 3: download file using file_token (NOT ticket) @@ -727,15 +747,20 @@ func (c *Client) downloadMediaFile(ctx context.Context, fileToken string) ([]byt // downloadRawBytes performs an authenticated GET and returns the raw response body. func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) { - token, err := c.GetTenantAccessToken(ctx) + _, err := c.GetTenantAccessToken(ctx) if err != nil { return nil, err } url := c.baseURL + path var lastErr error + gate := c.requestGate(http.MethodGet, path) for attempt := 0; attempt <= feishuMaxRetries; attempt++ { + token, err := c.waitForRequestToken(ctx, gate) + if err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("create download request: %w", err) @@ -760,15 +785,22 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err return nil, lastErr } - if resp.StatusCode == http.StatusTooManyRequests { + if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusBadRequest { body, _ := io.ReadAll(io.LimitReader(resp.Body, maxFeishuErrorPreviewBytes)) resp.Body.Close() - wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)]) - lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500)) + if !isFeishuRateLimited(resp.StatusCode, body) { + logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", + path, resp.StatusCode, truncate(string(body), 500)) + return nil, fmt.Errorf("download failed: status=%d body=%s", + resp.StatusCode, truncate(string(body), 500)) + } + wait := feishuRateLimitWait(resp.Header, time.Now()) + gate.cooldown(wait) + logger.Warnf(ctx, "[Feishu] download rate limited: api=%q wait=%s attempt=%d/%d", + gate.api, wait, attempt, feishuMaxRetries) + lastErr = fmt.Errorf("download rate limited: status=%d body=%s", + resp.StatusCode, truncate(string(body), 500)) if attempt < feishuMaxRetries { - if sErr := sleepCtx(ctx, wait); sErr != nil { - return nil, sErr - } continue } return nil, lastErr diff --git a/internal/datasource/connector/feishu/core/client_retry_test.go b/internal/datasource/connector/feishu/core/client_retry_test.go index 54baef72..a9a39dd9 100644 --- a/internal/datasource/connector/feishu/core/client_retry_test.go +++ b/internal/datasource/connector/feishu/core/client_retry_test.go @@ -14,9 +14,11 @@ import ( // retryTestServer builds a server that always answers the auth-token call and // routes the given target path to h, so tests can drive DoRequest's retry loop. -func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) { +func retryTestServer(t *testing.T, target string, h http.HandlerFunc) (*httptest.Server, *Config) { + t.Helper() + resetFeishuRequestGates(t) mux := http.NewServeMux() - mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, TokenResponse{ ApiResponse: ApiResponse{Code: 0}, TenantAccessToken: "fake-token", @@ -29,10 +31,10 @@ func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Conf } func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts == 1 { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if attempts.Load() == 1 { // "0" is coerced to a short delay inside the client so the test stays fast. w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) @@ -48,15 +50,15 @@ func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) { if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil { t.Fatalf("expected success after retry, got %v", err) } - if attempts < 2 { - t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts) + if attempts.Load() < 2 { + t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts.Load()) } } func TestDoRequest_429ExhaustsRetries(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) _, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`) @@ -68,15 +70,15 @@ func TestDoRequest_429ExhaustsRetries(t *testing.T) { if err == nil { t.Fatal("expected error when 429s exceed the retry budget") } - if attempts != 4 { // initial + 3 retries - t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts) + if attempts.Load() != 4 { // initial + 3 retries + t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts.Load()) } } func TestDoRequest_5xxRetriesOnce(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusInternalServerError) _, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`) }) @@ -89,15 +91,15 @@ func TestDoRequest_5xxRetriesOnce(t *testing.T) { if err := c.DoRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error after 5xx exhaustion") } - if attempts != 2 { // initial + 1 retry - t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts) + if attempts.Load() != 2 { // initial + 1 retry + t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts.Load()) } } func TestDoRequest_4xxNotRetried(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusBadRequest) _, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`) }) @@ -107,16 +109,16 @@ func TestDoRequest_4xxNotRetried(t *testing.T) { if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error on 400") } - if attempts != 1 { - t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts) + if attempts.Load() != 1 { + t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts.Load()) } } func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts == 1 { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/dl", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if attempts.Load() == 1 { w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) return @@ -134,15 +136,15 @@ func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) { if string(data) != "payload-bytes" { t.Errorf("data = %q, want %q", string(data), "payload-bytes") } - if attempts < 2 { - t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts) + if attempts.Load() < 2 { + t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts.Load()) } } func TestDownloadRawBytes_4xxNotRetried(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/dl", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusForbidden) }) defer ts.Close() @@ -151,33 +153,43 @@ func TestDownloadRawBytes_4xxNotRetried(t *testing.T) { if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil { t.Fatal("expected error on 403") } - if attempts != 1 { - t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts) + if attempts.Load() != 1 { + t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts.Load()) } } -func TestParseRetryAfter(t *testing.T) { - fallback := 5 * time.Second +func TestFeishuRateLimitWait(t *testing.T) { + now := time.Now().Truncate(time.Second) tests := []struct { - header string - want time.Duration + reset string + retry string + want time.Duration }{ - {"", fallback}, - {"0", 100 * time.Millisecond}, - {"-1", 100 * time.Millisecond}, // negative coerced to a short delay - {"3", 3 * time.Second}, - {"abc", fallback}, // unparseable + {"", "", 10 * time.Second}, + {"", "0", 100 * time.Millisecond}, + {"", "-1", 100 * time.Millisecond}, + {"3", "", 3 * time.Second}, + {"3", "5", 5 * time.Second}, + {"5", "3", 5 * time.Second}, + {"abc", "2", 2 * time.Second}, + {"", now.Add(4 * time.Second).UTC().Format(http.TimeFormat), 4 * time.Second}, + {"NaN", "Inf", 10 * time.Second}, + {"1e30", "abc", 10 * time.Second}, } + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "") for _, tt := range tests { - if got := parseRetryAfter(tt.header, fallback); got != tt.want { - t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want) + headers := http.Header{} + headers.Set("x-ogw-ratelimit-reset", tt.reset) + headers.Set("Retry-After", tt.retry) + if got := feishuRateLimitWait(headers, now); got != tt.want { + t.Errorf("wait(reset=%q, retry=%q) = %v, want %v", tt.reset, tt.retry, got, tt.want) } } } func TestDoRequest_RejectsOversizedResponse(t *testing.T) { var hits atomic.Int32 - ts, cfg := retryTestServer("/open-apis/docx/v1/documents", func(w http.ResponseWriter, _ *http.Request) { + ts, cfg := retryTestServer(t, "/open-apis/docx/v1/documents", func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) _, _ = w.Write(bytes.Repeat([]byte("a"), 1<<20+1)) }) @@ -252,7 +264,9 @@ func (t countingTransport) RoundTrip(req *http.Request) (*http.Response, error) // small bound and never copied whole into the returned error. func TestDownloadRawBytes_BoundsErrorBodies(t *testing.T) { const bodySize = 1 << 20 - for _, status := range []int{http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusForbidden} { + for _, status := range []int{ + http.StatusTooManyRequests, http.StatusBadRequest, http.StatusInternalServerError, http.StatusForbidden, + } { t.Run(http.StatusText(status), func(t *testing.T) { handler := func(w http.ResponseWriter, _ *http.Request) { if status == http.StatusTooManyRequests { @@ -261,7 +275,7 @@ func TestDownloadRawBytes_BoundsErrorBodies(t *testing.T) { w.WriteHeader(status) _, _ = w.Write(bytes.Repeat([]byte("a"), bodySize)) } - ts, cfg := retryTestServer("/open-apis/drive/v1/files/f/download", handler) + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/files/f/download", handler) defer ts.Close() c := NewClient(cfg) diff --git a/internal/datasource/connector/feishu/core/request_limits.go b/internal/datasource/connector/feishu/core/request_limits.go new file mode 100644 index 00000000..3b501ad4 --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_limits.go @@ -0,0 +1,238 @@ +package core + +import ( + "bytes" + "context" + "encoding/json" + "math" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultFeishuRequestsPerMinute = 80 + defaultFeishuRateLimitWait = 10 * time.Second + defaultFeishuMaxCooldown = 5 * time.Minute + defaultFeishuExportMaxPolls = 30 + defaultFeishuExportPollInterval = 2 * time.Second +) + +// Feishu defines this code in its API protocol. Derive the prefilter from the +// same value used for the decoded response so the two checks cannot drift. +const feishuRateLimitCode = 99991400 + +var feishuRateLimitCodeBytes = []byte(strconv.Itoa(feishuRateLimitCode)) + +// Export APIs each allow 100 requests/minute. Reserve headroom and pace requests +// without bursts. Wiki listing has its own budget; other APIs have separate +// reactive cooldowns by API path template. The registry spans clients, data +// sources and task retries in this process. Different App IDs and API hosts +// have independent budgets. +var feishuRequestGates sync.Map + +type feishuRequestGateKey struct { + baseURL string + appID string + api string +} + +type feishuRequestGate struct { + api string + mu sync.Mutex + interval time.Duration + nextRequest time.Time + blockedUntil time.Time +} + +func feishuRequestInterval() time.Duration { + rpm, err := strconv.Atoi(strings.TrimSpace(os.Getenv("FEISHU_API_REQUESTS_PER_MINUTE"))) + if err != nil || rpm <= 0 || int64(rpm) > int64(time.Minute) { + rpm = defaultFeishuRequestsPerMinute + } + return time.Minute / time.Duration(rpm) +} + +func feishuRequestAPI(method, path string) (string, bool) { + path, _, _ = strings.Cut(path, "?") + path = strings.TrimRight(path, "/") + if method == http.MethodPost && path == "/open-apis/drive/v1/export_tasks" { + return "export-create", true + } + if method == http.MethodGet && strings.HasPrefix(path, "/open-apis/drive/v1/export_tasks/") { + if strings.HasPrefix(path, "/open-apis/drive/v1/export_tasks/file/") && strings.HasSuffix(path, "/download") { + return "export-download", true + } + return "export-status", true + } + if method == http.MethodGet && + strings.HasPrefix(path, "/open-apis/wiki/v2/spaces/") && strings.HasSuffix(path, "/nodes") { + return "wiki-node-list", true + } + // Normalize the connector's remaining API paths without retaining document + // tokens, sheet ranges or query parameters. Methods have independent quotas. + parts := strings.Split(path, "/") + for _, template := range []string{ + "/open-apis/drive/v1/files/:token/download", + "/open-apis/drive/v1/medias/:token/download", + "/open-apis/drive/explorer/v2/folder/:token/meta", + "/open-apis/docx/v1/documents/:token/raw_content", + "/open-apis/docx/v1/documents/:token/blocks", + "/open-apis/sheets/v2/spreadsheets/:token/values/:range", + "/open-apis/bitable/v1/apps/:token/tables/:table/fields", + "/open-apis/bitable/v1/apps/:token/tables/:table/records/search", + "/open-apis/wiki/v2/spaces/get_node", + } { + pattern := strings.Split(template, "/") + if len(parts) != len(pattern) { + continue + } + match := true + for i, segment := range pattern { + if !strings.HasPrefix(segment, ":") && segment != parts[i] { + match = false + break + } + } + if match { + return method + " " + template, false + } + } + // Unknown endpoints share only their service/version/resource prefix, never + // a process-wide "other" gate or a gate per document token. + parts = strings.Split(strings.Trim(path, "/"), "/") + return method + " /" + strings.Join(parts[:min(4, len(parts))], "/"), false +} + +func feishuConfiguredDuration(name string, fallback time.Duration) time.Duration { + configured, err := time.ParseDuration(strings.TrimSpace(os.Getenv(name))) + if err == nil && configured > 0 { + return configured + } + return fallback +} + +func feishuExportMaxPolls() int { + configured, err := strconv.Atoi(strings.TrimSpace(os.Getenv("FEISHU_EXPORT_MAX_POLLS"))) + if err == nil && configured > 0 { + return configured + } + return defaultFeishuExportMaxPolls +} + +func (c *Client) requestGate(method, path string) *feishuRequestGate { + api, paced := feishuRequestAPI(method, path) + key := feishuRequestGateKey{strings.TrimRight(c.baseURL, "/"), c.appID, api} + if existing, ok := feishuRequestGates.Load(key); ok { + return existing.(*feishuRequestGate) + } + gate := &feishuRequestGate{api: api} + if paced { + gate.interval = feishuRequestInterval() + } + actual, _ := feishuRequestGates.LoadOrStore(key, gate) + return actual.(*feishuRequestGate) +} + +// wait reserves the next free slot before sleeping, so callers are granted in +// arrival order instead of racing for each slot after a shared wake-up. A +// cooldown imposed while sleeping invalidates the reservation; the caller waits +// it out and reserves again. Cancelled reservations are not reused, which only +// errs towards sending fewer requests. +func (g *feishuRequestGate) wait(ctx context.Context) error { + for { + if err := ctx.Err(); err != nil { + return err + } + g.mu.Lock() + now := time.Now() + if blocked := g.blockedUntil.Sub(now); blocked > 0 { + g.mu.Unlock() + if err := sleepCtx(ctx, blocked); err != nil { + return err + } + continue + } + slot := now + if g.nextRequest.After(slot) { + slot = g.nextRequest + } + g.nextRequest = slot.Add(g.interval) + g.mu.Unlock() + if err := sleepCtx(ctx, slot.Sub(now)); err != nil { + return err + } + g.mu.Lock() + blocked := g.blockedUntil.After(time.Now()) + g.mu.Unlock() + if !blocked { + return nil + } + } +} + +// Even an exhausted retry delays the next document, rather than letting a long +// sync continue hammering an API which has already rejected this application. +func (g *feishuRequestGate) cooldown(wait time.Duration) { + wait = min(wait, feishuConfiguredDuration("FEISHU_RATE_LIMIT_MAX_WAIT", defaultFeishuMaxCooldown)) + g.mu.Lock() + defer g.mu.Unlock() + until := time.Now().Add(wait) + if until.After(g.blockedUntil) { + g.blockedUntil = until + } +} + +func isFeishuRateLimited(status int, body []byte) bool { + if status == http.StatusTooManyRequests { + return true + } + if status != http.StatusBadRequest && status != http.StatusOK { + return false + } + // Successful block/record responses can be large. Only JSON-decode the + // envelope when it could contain the legacy rate-limit code. + if !bytes.Contains(body, feishuRateLimitCodeBytes) { + return false + } + var response struct { + Code int `json:"code"` + } + return json.Unmarshal(body, &response) == nil && response.Code == feishuRateLimitCode +} + +func feishuRateLimitWait(headers http.Header, now time.Time) time.Duration { + var wait time.Duration + valid := false + for _, name := range []string{"x-ogw-ratelimit-reset", "Retry-After"} { + value := strings.TrimSpace(headers.Get(name)) + if value == "" { + continue + } + seconds, err := strconv.ParseFloat(value, 64) + if err != nil && name == "Retry-After" { + if date, dateErr := http.ParseTime(value); dateErr == nil { + seconds, err = date.Sub(now).Seconds(), nil + } + } + if err != nil || math.IsNaN(seconds) || math.IsInf(seconds, 0) || + seconds >= float64(math.MaxInt64)/float64(time.Second) { + continue + } + valid = true + candidate := 100 * time.Millisecond + if seconds > 0 { + candidate = time.Duration(seconds * float64(time.Second)) + } + if candidate > wait { + wait = candidate + } + } + if !valid { + wait = feishuConfiguredDuration("FEISHU_RATE_LIMIT_RETRY_WAIT", defaultFeishuRateLimitWait) + } + return min(wait, feishuConfiguredDuration("FEISHU_RATE_LIMIT_MAX_WAIT", defaultFeishuMaxCooldown)) +} diff --git a/internal/datasource/connector/feishu/core/request_limits_test.go b/internal/datasource/connector/feishu/core/request_limits_test.go new file mode 100644 index 00000000..a2882df3 --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_limits_test.go @@ -0,0 +1,530 @@ +package core + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" +) + +func TestFeishuRequestIntervalConfiguration(t *testing.T) { + for _, tt := range []struct { + value string + want time.Duration + }{ + {"", 750 * time.Millisecond}, + {"80", 750 * time.Millisecond}, + {"60", time.Second}, + {"0", 750 * time.Millisecond}, + {"-1", 750 * time.Millisecond}, + {"invalid", 750 * time.Millisecond}, + } { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", tt.value) + if got := feishuRequestInterval(); got != tt.want { + t.Errorf("interval(%q)=%s, want %s", tt.value, got, tt.want) + } + } + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "7s") + if got := feishuRateLimitWait(http.Header{}, time.Now()); got != 7*time.Second { + t.Fatalf("configured fallback = %s", got) + } +} + +func TestFeishuGatesShareAppAndAPIIndependentOfDocument(t *testing.T) { + resetFeishuRequestGates(t) + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "80") + first := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test"}) + second := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test/"}) + poll := first.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/ticket-one?token=one") + if poll != second.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/ticket-two?token=two") { + t.Fatal("ticket and query must not create independent rate budgets") + } + if poll.interval != 750*time.Millisecond { + t.Fatalf("default interval = %s", poll.interval) + } + create := first.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") + download := first.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/file/one/download") + if create == poll || create == download || poll == download { + t.Fatal("export APIs must have separate budgets") + } + if download != second.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/file/two/download") { + t.Fatal("download tokens must share one budget") + } + otherApp := NewClient(&Config{AppID: t.Name() + "-other", BaseURL: first.baseURL}) + otherHost := NewClient(&Config{AppID: t.Name(), BaseURL: "http://other-host"}) + if create == otherApp.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") || + create == otherHost.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") { + t.Fatal("different applications and API hosts must not share a budget") + } + nodes := first.requestGate(http.MethodGet, "/open-apis/wiki/v2/spaces/one/nodes?parent_node_token=one") + if nodes != second.requestGate(http.MethodGet, "/open-apis/wiki/v2/spaces/two/nodes?parent_node_token=two") || + nodes.interval != poll.interval { + t.Fatal("wiki list paths must share a paced budget") + } +} + +func TestFeishuGatePacesConcurrentClients(t *testing.T) { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "3000") // 20ms for a bounded test. + var mu sync.Mutex + var arrivals []time.Time + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + arrivals = append(arrivals, time.Now()) + mu.Unlock() + writeJSON(w, ApiResponse{Code: 0}) + }) + defer ts.Close() + clients := []*Client{NewClient(cfg), NewClient(cfg)} + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + err := clients[i%2].DoRequest(context.Background(), + http.MethodPost, "/open-apis/drive/v1/export_tasks", nil, nil) + if err != nil { + t.Errorf("request: %v", err) + } + }(i) + } + wg.Wait() + sort.Slice(arrivals, func(i, j int) bool { return arrivals[i].Before(arrivals[j]) }) + if len(arrivals) != 6 { + t.Fatalf("received %d requests", len(arrivals)) + } + if elapsed := arrivals[5].Sub(arrivals[0]); elapsed < 90*time.Millisecond { + t.Fatalf("clients burst through shared budget: elapsed=%s", elapsed) + } +} + +func TestFeishuLegacyRateLimitRetriesJSONAndDownload(t *testing.T) { + for _, tt := range []struct { + name string + download bool + status int + }{ + {"json_400", false, http.StatusBadRequest}, + {"json_200", false, http.StatusOK}, + {"download_400", true, http.StatusBadRequest}, + } { + t.Run(tt.name, func(t *testing.T) { + var attempts atomic.Int32 + var firstAttempt atomic.Int64 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) == 1 { + firstAttempt.Store(time.Now().UnixNano()) + w.Header().Set("x-ogw-ratelimit-reset", "0.05") + w.WriteHeader(tt.status) + _, _ = io.WriteString(w, `{"code":99991400,"msg":"request trigger frequency limit"}`) + return + } + if time.Since(time.Unix(0, firstAttempt.Load())) < 50*time.Millisecond { + t.Error("request retried before Feishu reset time") + } + if tt.download { + _, _ = io.WriteString(w, "file-content") + } else { + writeJSON(w, ApiResponse{Code: 0}) + } + }) + defer ts.Close() + c := NewClient(cfg) + if tt.download { + data, err := c.downloadRawBytes(context.Background(), "/target") + if err != nil || string(data) != "file-content" { + t.Fatalf("download=%q err=%v", data, err) + } + } else if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err != nil { + t.Fatal(err) + } + if attempts.Load() != 2 { + t.Fatalf("attempts=%d, want 2", attempts.Load()) + } + }) + } +} + +func TestFeishuCooldownBlocksOtherClientAndHonorsCancellation(t *testing.T) { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.Header().Set("x-ogw-ratelimit-reset", "0.2") + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":99991400}`) + }) + defer ts.Close() + first := NewClient(cfg) + ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) + defer cancel() + if err := first.DoRequest(ctx, http.MethodGet, "/target", nil, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wait should be cancellable: %v", err) + } + second := NewClient(cfg) + ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel2() + if err := second.DoRequest(ctx2, http.MethodGet, "/target", nil, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("other client must inherit cooldown: %v", err) + } + if got := attempts.Load(); got != 1 { + t.Fatalf("cooldown allowed extra requests: %d", got) + } +} + +func TestFeishuRateLimitExhaustionProtectsNextDocument(t *testing.T) { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.Header().Set("Retry-After", "0.05") + w.WriteHeader(http.StatusTooManyRequests) + }) + defer ts.Close() + c := NewClient(cfg) + if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { + t.Fatal("persistent rate limit must not report success") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + err := NewClient(cfg).DoRequest(ctx, http.MethodGet, "/target", nil, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("next document must wait after exhausted retries: %v", err) + } + if got := attempts.Load(); got != 4 { + t.Fatalf("attempts=%d, want bounded 4", got) + } +} + +// Gate tests are serial (no t.Parallel): clear before and after each test so +// even a reused httptest port cannot inherit pacing or cooldown state. +func resetFeishuRequestGates(t *testing.T) { + t.Helper() + feishuRequestGates.Clear() + t.Cleanup(feishuRequestGates.Clear) +} + +func TestFeishuRequestGateReset(t *testing.T) { + config := &Config{AppID: t.Name(), BaseURL: "http://reused-test-host"} + path := "/open-apis/drive/v1/export_tasks" + old := NewClient(config).requestGate(http.MethodPost, path) + old.cooldown(time.Minute) + t.Run("reset", func(t *testing.T) { + resetFeishuRequestGates(t) + fresh := NewClient(config).requestGate(http.MethodPost, path) + if fresh == old || !fresh.blockedUntil.IsZero() || !fresh.nextRequest.IsZero() { + t.Fatal("reused host/app/API inherited gate state") + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := fresh.wait(ctx); err != nil { + t.Fatalf("fresh gate still blocked: %v", err) + } + }) + key := feishuRequestGateKey{config.BaseURL, config.AppID, "export-create"} + if _, exists := feishuRequestGates.Load(key); exists { + t.Fatal("test cleanup left a gate in the registry") + } +} + +func TestFeishuCooldownConfigurationAndBounds(t *testing.T) { + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "25ms") + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "1h") + if got := feishuRateLimitWait(http.Header{}, time.Now()); got != 25*time.Millisecond { + t.Fatalf("fallback escaped cap: %s", got) + } + headers := http.Header{"Retry-After": {"86400"}} + if got := feishuRateLimitWait(headers, time.Now()); got != 25*time.Millisecond { + t.Fatalf("header escaped configured cap: %s", got) + } + headers.Set("Retry-After", time.Now().Add(24*time.Hour).UTC().Format(http.TimeFormat)) + if got := feishuRateLimitWait(headers, time.Now()); got != 25*time.Millisecond { + t.Fatalf("HTTP date escaped configured cap: %s", got) + } + gate := &feishuRequestGate{} + gate.cooldown(time.Hour) + // Check the remaining cooldown, excluding any time the test was descheduled + // before cooldown() ran. Scheduler latency must not look like an excessive cap. + if remaining := time.Until(gate.blockedUntil); remaining > 25*time.Millisecond { + t.Fatalf("shared gate escaped cap: %s", remaining) + } + for _, invalid := range []string{"", "0s", "-1s", "bad"} { + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", invalid) + if got := feishuRateLimitWait(headers, time.Now()); got != 5*time.Minute { + t.Fatalf("invalid cap %q: got %s", invalid, got) + } + } +} + +func BenchmarkFeishuRateLimitSuccess(b *testing.B) { + body := append([]byte(`{"code":0,"data":"`), bytes.Repeat([]byte("x"), 16<<20)...) + body = append(body, []byte(`"}`)...) + b.SetBytes(int64(len(body))) + b.ResetTimer() + for b.Loop() { + if isFeishuRateLimited(http.StatusOK, body) { + b.Fatal("successful response classified as a rate limit") + } + } +} + +func TestFeishuReactiveCooldownsArePerAPI(t *testing.T) { + resetFeishuRequestGates(t) + c := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test"}) + pairs := [][2]string{ + {"/open-apis/drive/v1/files/one/download", "/open-apis/drive/v1/files/two/download?x=y"}, + {"/open-apis/drive/v1/medias/one/download", "/open-apis/drive/v1/medias/two/download"}, + {"/open-apis/docx/v1/documents/one/blocks", "/open-apis/docx/v1/documents/two/blocks?page_token=x"}, + {"/open-apis/docx/v1/documents/one/raw_content", "/open-apis/docx/v1/documents/two/raw_content"}, + {"/open-apis/sheets/v2/spreadsheets/one/values/A1", "/open-apis/sheets/v2/spreadsheets/two/values/B2"}, + {"/open-apis/bitable/v1/apps/one/tables/t1/fields", "/open-apis/bitable/v1/apps/two/tables/t2/fields"}, + { + "/open-apis/bitable/v1/apps/one/tables/t1/records/search", + "/open-apis/bitable/v1/apps/two/tables/t2/records/search", + }, + {"/open-apis/drive/explorer/v2/folder/one/meta", "/open-apis/drive/explorer/v2/folder/two/meta"}, + {"/open-apis/wiki/v2/spaces/get_node?token=one", "/open-apis/wiki/v2/spaces/get_node?token=two"}, + {"/open-apis/wiki/v2/spaces?page_token=one", "/open-apis/wiki/v2/spaces?page_token=two"}, + } + seen := make(map[*feishuRequestGate]bool) + for _, pair := range pairs { + gate := c.requestGate(http.MethodGet, pair[0]) + if gate != c.requestGate(http.MethodGet, pair[1]) || gate.interval != 0 { + t.Fatalf("reactive gate must normalize tokens and query: %v", pair) + } + if seen[gate] { + t.Fatalf("unrelated APIs share a cooldown: %v", pair) + } + seen[gate] = true + if gate == c.requestGate(http.MethodPost, pair[0]) { + t.Fatalf("HTTP methods share a cooldown: %v", pair) + } + } + media := c.requestGate(http.MethodGet, pairs[1][0]) + media.cooldown(time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := c.requestGate(http.MethodGet, pairs[2][0]).wait(ctx); err != nil { + t.Fatalf("media cooldown blocked docx: %v", err) + } + if err := media.wait(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("media did not retain its own cooldown: %v", err) + } +} + +func TestFeishuRateLimitDetection(t *testing.T) { + for _, tt := range []struct { + status int + body string + want bool + }{ + {http.StatusOK, `{"code":0,"data":{"text":"99991400"}}`, false}, + {http.StatusOK, `{"code":99991400}`, true}, + {http.StatusBadRequest, `{"code":99991400}`, true}, + {http.StatusOK, `{"code":0}`, false}, + {http.StatusOK, `invalid 99991400`, false}, + {http.StatusForbidden, `{"code":99991400}`, false}, + {http.StatusTooManyRequests, ``, true}, + } { + if got := isFeishuRateLimited(tt.status, []byte(tt.body)); got != tt.want { + t.Errorf("rate limited(%d, %s) = %v, want %v", tt.status, tt.body, got, tt.want) + } + } +} + +// Run HTTP handlers in process so synctest can advance time without real sockets. +type feishuHandlerTransport struct { + http.Handler +} + +func (tr feishuHandlerTransport) RoundTrip(r *http.Request) (*http.Response, error) { + recorder := httptest.NewRecorder() + tr.ServeHTTP(recorder, r) + return recorder.Result(), nil +} + +func TestFeishuExportPollingRetainsTicketAcrossCooldown(t *testing.T) { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "80") + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "70s") + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "5m") + t.Setenv("FEISHU_EXPORT_MAX_POLLS", "2") + t.Setenv("FEISHU_EXPORT_POLL_INTERVAL", "2s") + synctest.Test(t, func(t *testing.T) { + resetFeishuRequestGates(t) + var creates, polls, downloads int + mux := http.NewServeMux() + mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, _ *http.Request) { + creates++ + _, _ = io.WriteString(w, `{"code":0,"data":{"ticket":"same-ticket"}}`) + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/same-ticket", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("token") != "document" { + t.Error("poll did not retain the document token") + } + polls++ + switch polls { + case 1: + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":99991400}`) + case 2: + _, _ = io.WriteString(w, `{"code":0,"data":{"result":{"job_status":2}}}`) + default: + _, _ = io.WriteString(w, + `{"code":0,"data":{"result":{"job_status":0,"file_token":"file","file_name":"doc.pdf"}}}`) + } + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/file/download", + func(w http.ResponseWriter, _ *http.Request) { + downloads++ + _, _ = io.WriteString(w, "export-content") + }) + cfg := &Config{AppID: t.Name(), BaseURL: "http://feishu-export.test"} + c := NewClient(cfg) + c.httpClient.Transport = feishuHandlerTransport{mux} + c.tokenCache, c.tokenExpAt = "cached-token", time.Now().Add(time.Hour) + // Both shared and in-poll cooldowns exceed the previous 60-second deadline. + started := time.Now() + other := NewClient(cfg) + other.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/other-ticket").cooldown(70 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + data, name, err := c.ExportAndDownload(ctx, "document", "docx") + if err != nil || name != "doc.pdf" || string(data) != "export-content" { + t.Fatalf("export across cooldown: data=%q name=%q err=%v", data, name, err) + } + if creates != 1 || polls != 3 || downloads != 1 { + t.Fatalf("create/poll/download=%d/%d/%d, want 1/3/1", creates, polls, downloads) + } + if elapsed := time.Since(started); elapsed < 140*time.Second { + t.Fatalf("export skipped the shared or in-poll cooldown: %s", elapsed) + } + }) +} + +func TestFeishuExportPollingIsBoundedAndCancellable(t *testing.T) { + for _, cancelWhileWaiting := range []bool{false, true} { + t.Run(fmt.Sprint(cancelWhileWaiting), func(t *testing.T) { + t.Setenv("FEISHU_EXPORT_MAX_POLLS", "3") + t.Setenv("FEISHU_EXPORT_POLL_INTERVAL", "1ms") + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "60000") + var creates, polls atomic.Int32 + createHandler := func(w http.ResponseWriter, _ *http.Request) { + creates.Add(1) + _, _ = io.WriteString(w, `{"code":0,"data":{"ticket":"pending"}}`) + } + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/export_tasks", createHandler) + defer ts.Close() + ts.Config.Handler.(*http.ServeMux).HandleFunc("/open-apis/drive/v1/export_tasks/pending", + func(w http.ResponseWriter, _ *http.Request) { + polls.Add(1) + _, _ = io.WriteString(w, `{"code":0,"data":{"result":{"job_status":2}}}`) + }) + c := NewClient(cfg) + if cancelWhileWaiting { + c.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/pending").cooldown(time.Second) + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, _, err := c.ExportAndDownload(ctx, "document", "docx") + if cancelWhileWaiting { + if !errors.Is(err, context.DeadlineExceeded) || polls.Load() != 0 { + t.Fatalf("polling ignored cancellation: polls=%d err=%v", polls.Load(), err) + } + } else if err == nil || !strings.Contains(err.Error(), "after 3 polls") || polls.Load() != 3 { + t.Fatalf("polling is unbounded: polls=%d err=%v", polls.Load(), err) + } + // Exercise the same wrapping and metadata path used by the connectors. + metadata := FeishuErrorItemMeta(fmt.Errorf("export document: %w", err), nil) + if metadata["error_reason_code"] != "feishu_timeout" { + t.Fatalf("polling lost its timeout classification: %v", metadata) + } + if strings.Contains(metadata["error_reason"], "ticket=") { + t.Fatal("export ticket leaked into the user-facing failure reason") + } + if creates.Load() != 1 { + t.Fatalf("created %d tasks, want 1", creates.Load()) + } + }) + } +} + +func TestFeishuRefreshesTokenAfterCooldown(t *testing.T) { + for _, download := range []bool{false, true} { + t.Run(fmt.Sprint(download), func(t *testing.T) { + resetFeishuRequestGates(t) + var auths, attempts atomic.Int32 + var c *Client + mux := http.NewServeMux() + authHandler := func(w http.ResponseWriter, _ *http.Request) { + token := fmt.Sprintf("token-%d", auths.Add(1)) + writeJSON(w, TokenResponse{TenantAccessToken: token, Expire: 7200}) + } + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", authHandler) + mux.HandleFunc("/target", func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) == 1 { + c.tokenMu.Lock() + c.tokenExpAt = time.Now().Add(5 * time.Millisecond) + c.tokenMu.Unlock() + w.Header().Set("Retry-After", "0.03") + w.WriteHeader(http.StatusTooManyRequests) + return + } + if r.Header.Get("Authorization") != "Bearer token-2" { + t.Error("request reused the token that expired during cooldown") + } + _, _ = io.WriteString(w, `{"code":0}`) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + c = NewClient(&Config{AppID: t.Name(), BaseURL: ts.URL}) + var err error + if download { + _, err = c.downloadRawBytes(context.Background(), "/target") + } else { + err = c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil) + } + if err != nil || auths.Load() != 2 || attempts.Load() != 2 { + t.Fatalf("auth/attempts=%d/%d err=%v", auths.Load(), attempts.Load(), err) + } + }) + } +} + +func TestFeishuGateGrantsInArrivalOrder(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + gate := &feishuRequestGate{interval: time.Second} + start := time.Now() + var mu sync.Mutex + var order []int + var grants []time.Duration + var wg sync.WaitGroup + for i := range 5 { + wg.Add(1) + go func() { + defer wg.Done() + if err := gate.wait(context.Background()); err != nil { + t.Error(err) + return + } + mu.Lock() + order = append(order, i) + grants = append(grants, time.Since(start)) + mu.Unlock() + }() + synctest.Wait() + } + wg.Wait() + for i := range 5 { + if order[i] != i || grants[i] != time.Duration(i)*time.Second { + t.Fatalf("grants out of arrival order or pace: order=%v grants=%v", order, grants) + } + } + }) +} diff --git a/internal/datasource/connector/feishu/core/request_token_test.go b/internal/datasource/connector/feishu/core/request_token_test.go new file mode 100644 index 00000000..690ac54a --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_token_test.go @@ -0,0 +1,121 @@ +package core + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sort" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Cached-token requests first check Err when entering gate.wait. Observe that +// boundary so the test expires the token only after every caller authenticated. +type feishuGateWaitContext struct { + context.Context + once sync.Once + entered chan<- struct{} +} + +func (c *feishuGateWaitContext) Err() error { + c.once.Do(func() { c.entered <- struct{}{} }) + return c.Context.Err() +} + +func TestFeishuTokenRefreshPreservesPacingAndCooldown(t *testing.T) { + for _, mode := range []string{"json", "download"} { + t.Run(mode, func(t *testing.T) { + resetFeishuRequestGates(t) + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "600") + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "5m") + const count = 4 + const interval = 100 * time.Millisecond + var mu sync.Mutex + var arrivals []time.Time + var earliestRequest time.Time + var auths atomic.Int32 + var gate *feishuRequestGate + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" { + auths.Add(1) + // Slow enough for every caller to pass the gate in the broken + // implementation and queue behind the refreshing caller's tokenMu. + time.Sleep(time.Duration(count+1) * interval) + mu.Lock() + earliestRequest = time.Now().Add(3 * interval) + mu.Unlock() + // A concurrent data source can extend this shared cooldown while + // our token is refreshing. All callers must honor the new delay. + gate.cooldown(3 * interval) + writeJSON(w, TokenResponse{TenantAccessToken: "fresh-token", Expire: 7200}) + return + } + if r.Header.Get("Authorization") != "Bearer fresh-token" { + t.Error("request sent the token that expired while waiting") + } + mu.Lock() + arrivals = append(arrivals, time.Now()) + mu.Unlock() + _, _ = io.WriteString(w, `{"code":0}`) + })) + defer server.Close() + c := NewClient(&Config{AppID: t.Name(), BaseURL: server.URL}) + c.tokenCache = "cached-token" + c.tokenExpAt = time.Now().Add(time.Hour) + method, path := http.MethodPost, "/open-apis/drive/v1/export_tasks" + if mode == "download" { + method, path = http.MethodGet, "/open-apis/drive/v1/export_tasks/file/export-token/download" + } + gate = c.requestGate(method, path) + gate.cooldown(500 * time.Millisecond) + entered := make(chan struct{}, count) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var wg sync.WaitGroup + for range count { + wg.Add(1) + go func() { + defer wg.Done() + observed := &feishuGateWaitContext{Context: ctx, entered: entered} + var err error + if mode == "download" { + _, err = c.downloadRawBytes(observed, path) + } else { + err = c.DoRequest(observed, method, path, nil, nil) + } + if err != nil { + t.Errorf("request: %v", err) + } + }() + } + for range count { + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("requests failed to reach the gate") + } + } + c.tokenMu.Lock() + c.tokenExpAt = time.Now().Add(-time.Second) + c.tokenMu.Unlock() + wg.Wait() + mu.Lock() + defer mu.Unlock() + if len(arrivals) != count || auths.Load() != 1 { + t.Fatalf("requests/refreshes=%d/%d, want %d/1", len(arrivals), auths.Load(), count) + } + sort.Slice(arrivals, func(i, j int) bool { return arrivals[i].Before(arrivals[j]) }) + if arrivals[0].Before(earliestRequest) { + t.Errorf("request ignored the cooldown imposed during token refresh") + } + for i := 1; i < len(arrivals); i++ { + if gap := arrivals[i].Sub(arrivals[i-1]); gap < interval/2 { + t.Errorf("requests burst after token refresh: gap=%s, interval=%s", gap, interval) + } + } + }) + } +} diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.go index fb6cbb00..eec1cce5 100644 --- a/internal/datasource/connector/feishu/core/shared.go +++ b/internal/datasource/connector/feishu/core/shared.go @@ -3,6 +3,7 @@ package core import ( "context" "encoding/json" + "errors" "fmt" "maps" "net/http" @@ -105,7 +106,8 @@ func feishuFailure(err error) (code, codeValue, fallback string) { return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes" case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"): return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync" - case strings.Contains(s, "timed out"), + case errors.Is(err, errExportPollsExhausted), + strings.Contains(s, "timed out"), strings.Contains(s, "timeout"), strings.Contains(s, "deadline exceeded"): return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"