diff --git a/.env.example b/.env.example index 1248e3c3..59ae4ddf 100644 --- a/.env.example +++ b/.env.example @@ -551,6 +551,18 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # 此方案为临时方案,若后续有更好的解析方案,会移除此环境变量并替换。 # FEISHU_DOCX_PARSE_MODE=export +# 飞书导出创建 / 查询 / 下载、Wiki 子节点列表分别按应用共享配额(单进程)。 +# 默认每个接口 80 次/分钟,平滑发送、不积累突发额度;正整数可覆盖默认值。 +# FEISHU_API_REQUESTS_PER_MINUTE=80 +# 限流响应没有有效等待提示时的等待时长(Go duration,默认 10s)。 +# FEISHU_RATE_LIMIT_RETRY_WAIT=10s +# 每次共享冷却的最大等待时长,包含响应头提示和上述默认等待(Go duration)。 +# FEISHU_RATE_LIMIT_MAX_WAIT=5m +# 导出任务最多轮询次数;限流重试及排队不消耗轮询次数,任务上下文仍可取消。 +# FEISHU_EXPORT_MAX_POLLS=30 +# 导出任务两次轮询之间的间隔(Go duration)。 +# FEISHU_EXPORT_POLL_INTERVAL=2s + # ##################################################################### # F. 认证与空间隔离 # ##################################################################### diff --git a/internal/datasource/README.md b/internal/datasource/README.md index 9512515d..0b9b2bcb 100644 --- a/internal/datasource/README.md +++ b/internal/datasource/README.md @@ -234,11 +234,66 @@ GET /api/v1/datasource/types # Available connectors 5. **Multi-tenant**: All operations are tenant-isolated 6. **Channel Tracking**: Knowledge.Channel field tracks data source type for origin tracking +## Feishu request pacing and rate-limit retries + +Feishu export creation, export status polling, export downloads, and Wiki node +listing each use a separate request budget. Clients using the same API host and +App ID share these budgets within one backend process, including concurrent data +sources and task retries. Document tokens, export tickets, and query parameters +do not create additional budgets. + +| Environment variable | Default | Behavior | +| --- | --- | --- | +| `FEISHU_API_REQUESTS_PER_MINUTE` | `80` | Positive integer; requests to each paced API are evenly spaced (750 ms by default), without accumulating burst capacity. | +| `FEISHU_RATE_LIMIT_RETRY_WAIT` | `10s` | Positive Go duration used when the response supplies no valid reset/retry hint. | +| `FEISHU_RATE_LIMIT_MAX_WAIT` | `5m` | Positive Go duration capping each cooldown, including response-header hints and the fallback. | +| `FEISHU_EXPORT_MAX_POLLS` | `30` | Positive integer bounding status polls of the same export task; API retries do not consume additional polls. | +| `FEISHU_EXPORT_POLL_INTERVAL` | `2s` | Positive Go duration between status polls while the export is still processing. | + +Invalid values use the defaults. Other Feishu APIs retain their existing request +frequency, with reactive cooldowns scoped to the HTTP method and normalized API +path template. Document tokens, table IDs, sheet ranges, and query parameters do +not split a known API's cooldown. Media downloads, file downloads, docx blocks, +raw content, bitable fields/record searches, sheet values, folder metadata, Wiki +space listing, and Wiki node lookup use independent cooldowns. Unknown endpoints +use their service/version/resource prefix. Check the application's actual API +quotas before increasing the configured budget. + +JSON requests retry HTTP 429 and HTTP 400/200 responses containing Feishu error +code `99991400`. Downloads retry HTTP 429 and HTTP 400 with that error code. +The wait is the longer valid value from `x-ogw-ratelimit-reset` and `Retry-After`; +`Retry-After` also accepts an HTTP date. All hints are clamped to the configured +maximum so an unreasonable gateway header cannot indefinitely block later syncs. +Without a valid hint, a rate-limit response imposes a 10-second cooldown by +default. The three retries add a nominal 30 seconds of cooldown; shared pacing +or further rate-limit responses from other clients can extend the total wait. +This replaces the original 2/4/8-second backoff with a shared cooldown scoped +to the affected API. +Only callers of the same API share that cooldown. Requests retain the existing +limit of three retries. Even when retries are exhausted, subsequent documents and other +clients sharing that API wait for the cooldown. Waiting honors task cancellation. +Ordinary permission/validation errors still fail immediately, and download error +bodies retain their existing size bounds. + +Export polling uses a bounded number of polls rather than a 60-second wall-clock +deadline. Shared pacing and cooldown waits therefore do not cause false export +timeouts or create replacement tasks within the operation. With the defaults, +an export that remains processing is checked up to 30 times, with 2 seconds +between polls. The caller's context bounds total elapsed time, including waits +and retries. Exhausting the poll budget retains the `feishu_timeout` failure +classification. Tokens are checked again after gate waits; requests needing a +token refresh rejoin the gate afterwards, so both JSON requests and downloads +remain paced and honor any cooldown imposed during the refresh. + +These budgets are process-local. Multiple backend instances or other applications +using the same Feishu credentials may still hit the upstream quota; configure +their combined traffic accordingly. + ## Future Enhancements 1. **Webhook Support**: Real-time push from platforms that support webhooks 2. **Conflict Resolution**: Advanced merge/conflict strategies for overlapping content -3. **Rate Limiting**: Per-connector rate limiting and backoff strategies +3. **Rate Limiting**: Extend request pacing to other connectors and coordinate quotas across backend instances 4. **Scheduling**: Full cron scheduler with time zone support 5. **Monitoring**: Metrics, alerting, and sync health dashboards 6. **Filtering**: User-defined filters for selective syncing (by title, date, tags, etc) diff --git a/internal/datasource/connector/feishu/core/client.go b/internal/datasource/connector/feishu/core/client.go index 3fb3dbcf..d8eef921 100644 --- a/internal/datasource/connector/feishu/core/client.go +++ b/internal/datasource/connector/feishu/core/client.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "sync" "time" @@ -166,8 +165,33 @@ func (c *Client) GetTenantAccessToken(ctx context.Context) (string, error) { return c.tokenCache, nil } +// waitForRequestToken authenticates before pacing and rechecks the cached token +// afterwards. A refresh must happen before rejoining the gate, otherwise requests +// that have already been paced can pile up behind tokenMu and then burst together. +func (c *Client) waitForRequestToken(ctx context.Context, gate *feishuRequestGate) (string, error) { + for { + token, err := c.GetTenantAccessToken(ctx) + if err != nil { + return "", err + } + if err := gate.wait(ctx); err != nil { + return "", err + } + // Never wait on a token refresh after the gate has granted a request slot. + if !c.tokenMu.TryLock() { + continue + } + valid := token == c.tokenCache && time.Now().Before(c.tokenExpAt) + c.tokenMu.Unlock() + if valid { + return token, nil + } + } +} + // Retry policy shared by DoRequest (JSON API calls) and downloadRawBytes (file -// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off. +// downloads): rate limits honour Feishu's reset header / Retry-After, 5xx +// retries once, transport errors back off. const ( feishuMaxRetries = 3 feishuMax5xxRetries = 1 @@ -188,8 +212,8 @@ var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * t // retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive // export/wiki APIs are aggressively rate limited, and a thousand-document sync // issues tens of thousands of calls; without backoff a single 429 burst used to -// fail whole swathes of documents silently. 429 responses honour Retry-After; -// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx). +// fail whole swathes of documents silently. HTTP 400 with code 99991400 is +// also a rate limit. Ordinary permission/validation errors still fail fast. func (c *Client) DoRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error { const ( maxRetries = feishuMaxRetries @@ -198,7 +222,7 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf ) backoff := feishuRetryBackoff - token, err := c.GetTenantAccessToken(ctx) + _, err := c.GetTenantAccessToken(ctx) if err != nil { return err } @@ -213,8 +237,13 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf url := c.baseURL + path var lastErr error + gate := c.requestGate(method, path) for attempt := 0; attempt <= maxRetries; attempt++ { + token, err := c.waitForRequestToken(ctx, gate) + if err != nil { + return err + } var bodyReader io.Reader if bodyBytes != nil { bodyReader = bytes.NewReader(bodyBytes) @@ -260,13 +289,14 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s", method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000)) - if resp.StatusCode == http.StatusTooManyRequests { - wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)]) - lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500)) + if isFeishuRateLimited(resp.StatusCode, respBody) { + wait := feishuRateLimitWait(resp.Header, time.Now()) + gate.cooldown(wait) + logger.Warnf(ctx, "[Feishu] rate limited: api=%q wait=%s attempt=%d/%d", + gate.api, wait, attempt, maxRetries) + lastErr = fmt.Errorf("feishu rate limited: status=%d body=%s", + resp.StatusCode, truncate(string(respBody), 500)) if attempt < maxRetries { - if sErr := sleepCtx(ctx, wait); sErr != nil { - return sErr - } continue } return lastErr @@ -298,23 +328,6 @@ func (c *Client) DoRequest(ctx context.Context, method, path string, body interf return lastErr } -// parseRetryAfter interprets a Retry-After header value (seconds) into a wait -// duration, coercing 0/negative to a short delay and falling back when absent -// or unparseable. -func parseRetryAfter(header string, fallback time.Duration) time.Duration { - if header == "" { - return fallback - } - secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64) - if err != nil { - return fallback - } - if secs <= 0 { - return 100 * time.Millisecond - } - return time.Duration(secs * float64(time.Second)) -} - // sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the // context ends first so retries abort promptly on task cancellation/timeout. func sleepCtx(ctx context.Context, d time.Duration) error { @@ -646,10 +659,15 @@ func (c *Client) downloadExportFile(ctx context.Context, fileToken string) ([]by return c.downloadRawBytes(ctx, path) } +// errExportPollsExhausted keeps the error classification independent of the +// message's poll count and ticket details, including when connectors wrap it. +var errExportPollsExhausted = errors.New("export task still processing") + // ExportAndDownload is a high-level helper that creates an export task, polls until // completion, and downloads the resulting file. Returns (fileBytes, fileName, error). // -// Timeout: 60 seconds. Poll interval: 2 seconds. +// Poll count and interval are configurable. Pacing/cooldown waits do not consume +// the poll budget; the caller's context still bounds the entire operation. func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) { // Determine export format fileExt, ok := ObjTypeToExportFileExtension[objType] @@ -668,11 +686,13 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string return nil, "", err } - // Step 2: poll until ready (max 60s, every 2s) - deadline := time.Now().Add(60 * time.Second) + // Step 2: retain the same ticket while waiting on shared API limits. A wall + // clock deadline here would turn other documents' cooldowns into false failures. + maxPolls := feishuExportMaxPolls() + pollInterval := feishuConfiguredDuration("FEISHU_EXPORT_POLL_INTERVAL", defaultFeishuExportPollInterval) var fileToken, fileName string - for time.Now().Before(deadline) { + for poll := 0; poll < maxPolls; poll++ { fileToken, fileName, err = c.getExportTaskStatus(ctx, ticket, objToken) if err != nil { return nil, "", err @@ -680,15 +700,15 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string if fileToken != "" { break // export ready } - select { - case <-ctx.Done(): - return nil, "", ctx.Err() - case <-time.After(2 * time.Second): + if poll+1 < maxPolls { + if err := sleepCtx(ctx, pollInterval); err != nil { + return nil, "", err + } } } if fileToken == "" { - return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket) + return nil, "", fmt.Errorf("%w after %d polls (ticket=%s)", errExportPollsExhausted, maxPolls, ticket) } // Step 3: download file using file_token (NOT ticket) @@ -727,15 +747,20 @@ func (c *Client) downloadMediaFile(ctx context.Context, fileToken string) ([]byt // downloadRawBytes performs an authenticated GET and returns the raw response body. func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) { - token, err := c.GetTenantAccessToken(ctx) + _, err := c.GetTenantAccessToken(ctx) if err != nil { return nil, err } url := c.baseURL + path var lastErr error + gate := c.requestGate(http.MethodGet, path) for attempt := 0; attempt <= feishuMaxRetries; attempt++ { + token, err := c.waitForRequestToken(ctx, gate) + if err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("create download request: %w", err) @@ -760,15 +785,22 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err return nil, lastErr } - if resp.StatusCode == http.StatusTooManyRequests { + if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusBadRequest { body, _ := io.ReadAll(io.LimitReader(resp.Body, maxFeishuErrorPreviewBytes)) resp.Body.Close() - wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)]) - lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500)) + if !isFeishuRateLimited(resp.StatusCode, body) { + logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", + path, resp.StatusCode, truncate(string(body), 500)) + return nil, fmt.Errorf("download failed: status=%d body=%s", + resp.StatusCode, truncate(string(body), 500)) + } + wait := feishuRateLimitWait(resp.Header, time.Now()) + gate.cooldown(wait) + logger.Warnf(ctx, "[Feishu] download rate limited: api=%q wait=%s attempt=%d/%d", + gate.api, wait, attempt, feishuMaxRetries) + lastErr = fmt.Errorf("download rate limited: status=%d body=%s", + resp.StatusCode, truncate(string(body), 500)) if attempt < feishuMaxRetries { - if sErr := sleepCtx(ctx, wait); sErr != nil { - return nil, sErr - } continue } return nil, lastErr diff --git a/internal/datasource/connector/feishu/core/client_retry_test.go b/internal/datasource/connector/feishu/core/client_retry_test.go index 54baef72..a9a39dd9 100644 --- a/internal/datasource/connector/feishu/core/client_retry_test.go +++ b/internal/datasource/connector/feishu/core/client_retry_test.go @@ -14,9 +14,11 @@ import ( // retryTestServer builds a server that always answers the auth-token call and // routes the given target path to h, so tests can drive DoRequest's retry loop. -func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) { +func retryTestServer(t *testing.T, target string, h http.HandlerFunc) (*httptest.Server, *Config) { + t.Helper() + resetFeishuRequestGates(t) mux := http.NewServeMux() - mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, TokenResponse{ ApiResponse: ApiResponse{Code: 0}, TenantAccessToken: "fake-token", @@ -29,10 +31,10 @@ func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Conf } func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts == 1 { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if attempts.Load() == 1 { // "0" is coerced to a short delay inside the client so the test stays fast. w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) @@ -48,15 +50,15 @@ func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) { if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil { t.Fatalf("expected success after retry, got %v", err) } - if attempts < 2 { - t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts) + if attempts.Load() < 2 { + t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts.Load()) } } func TestDoRequest_429ExhaustsRetries(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) _, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`) @@ -68,15 +70,15 @@ func TestDoRequest_429ExhaustsRetries(t *testing.T) { if err == nil { t.Fatal("expected error when 429s exceed the retry budget") } - if attempts != 4 { // initial + 3 retries - t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts) + if attempts.Load() != 4 { // initial + 3 retries + t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts.Load()) } } func TestDoRequest_5xxRetriesOnce(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusInternalServerError) _, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`) }) @@ -89,15 +91,15 @@ func TestDoRequest_5xxRetriesOnce(t *testing.T) { if err := c.DoRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error after 5xx exhaustion") } - if attempts != 2 { // initial + 1 retry - t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts) + if attempts.Load() != 2 { // initial + 1 retry + t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts.Load()) } } func TestDoRequest_4xxNotRetried(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusBadRequest) _, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`) }) @@ -107,16 +109,16 @@ func TestDoRequest_4xxNotRetried(t *testing.T) { if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error on 400") } - if attempts != 1 { - t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts) + if attempts.Load() != 1 { + t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts.Load()) } } func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts == 1 { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/dl", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if attempts.Load() == 1 { w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) return @@ -134,15 +136,15 @@ func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) { if string(data) != "payload-bytes" { t.Errorf("data = %q, want %q", string(data), "payload-bytes") } - if attempts < 2 { - t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts) + if attempts.Load() < 2 { + t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts.Load()) } } func TestDownloadRawBytes_4xxNotRetried(t *testing.T) { - var attempts int - ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) { - attempts++ + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/dl", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) w.WriteHeader(http.StatusForbidden) }) defer ts.Close() @@ -151,33 +153,43 @@ func TestDownloadRawBytes_4xxNotRetried(t *testing.T) { if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil { t.Fatal("expected error on 403") } - if attempts != 1 { - t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts) + if attempts.Load() != 1 { + t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts.Load()) } } -func TestParseRetryAfter(t *testing.T) { - fallback := 5 * time.Second +func TestFeishuRateLimitWait(t *testing.T) { + now := time.Now().Truncate(time.Second) tests := []struct { - header string - want time.Duration + reset string + retry string + want time.Duration }{ - {"", fallback}, - {"0", 100 * time.Millisecond}, - {"-1", 100 * time.Millisecond}, // negative coerced to a short delay - {"3", 3 * time.Second}, - {"abc", fallback}, // unparseable + {"", "", 10 * time.Second}, + {"", "0", 100 * time.Millisecond}, + {"", "-1", 100 * time.Millisecond}, + {"3", "", 3 * time.Second}, + {"3", "5", 5 * time.Second}, + {"5", "3", 5 * time.Second}, + {"abc", "2", 2 * time.Second}, + {"", now.Add(4 * time.Second).UTC().Format(http.TimeFormat), 4 * time.Second}, + {"NaN", "Inf", 10 * time.Second}, + {"1e30", "abc", 10 * time.Second}, } + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "") for _, tt := range tests { - if got := parseRetryAfter(tt.header, fallback); got != tt.want { - t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want) + headers := http.Header{} + headers.Set("x-ogw-ratelimit-reset", tt.reset) + headers.Set("Retry-After", tt.retry) + if got := feishuRateLimitWait(headers, now); got != tt.want { + t.Errorf("wait(reset=%q, retry=%q) = %v, want %v", tt.reset, tt.retry, got, tt.want) } } } func TestDoRequest_RejectsOversizedResponse(t *testing.T) { var hits atomic.Int32 - ts, cfg := retryTestServer("/open-apis/docx/v1/documents", func(w http.ResponseWriter, _ *http.Request) { + ts, cfg := retryTestServer(t, "/open-apis/docx/v1/documents", func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) _, _ = w.Write(bytes.Repeat([]byte("a"), 1<<20+1)) }) @@ -252,7 +264,9 @@ func (t countingTransport) RoundTrip(req *http.Request) (*http.Response, error) // small bound and never copied whole into the returned error. func TestDownloadRawBytes_BoundsErrorBodies(t *testing.T) { const bodySize = 1 << 20 - for _, status := range []int{http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusForbidden} { + for _, status := range []int{ + http.StatusTooManyRequests, http.StatusBadRequest, http.StatusInternalServerError, http.StatusForbidden, + } { t.Run(http.StatusText(status), func(t *testing.T) { handler := func(w http.ResponseWriter, _ *http.Request) { if status == http.StatusTooManyRequests { @@ -261,7 +275,7 @@ func TestDownloadRawBytes_BoundsErrorBodies(t *testing.T) { w.WriteHeader(status) _, _ = w.Write(bytes.Repeat([]byte("a"), bodySize)) } - ts, cfg := retryTestServer("/open-apis/drive/v1/files/f/download", handler) + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/files/f/download", handler) defer ts.Close() c := NewClient(cfg) diff --git a/internal/datasource/connector/feishu/core/request_limits.go b/internal/datasource/connector/feishu/core/request_limits.go new file mode 100644 index 00000000..3b501ad4 --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_limits.go @@ -0,0 +1,238 @@ +package core + +import ( + "bytes" + "context" + "encoding/json" + "math" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultFeishuRequestsPerMinute = 80 + defaultFeishuRateLimitWait = 10 * time.Second + defaultFeishuMaxCooldown = 5 * time.Minute + defaultFeishuExportMaxPolls = 30 + defaultFeishuExportPollInterval = 2 * time.Second +) + +// Feishu defines this code in its API protocol. Derive the prefilter from the +// same value used for the decoded response so the two checks cannot drift. +const feishuRateLimitCode = 99991400 + +var feishuRateLimitCodeBytes = []byte(strconv.Itoa(feishuRateLimitCode)) + +// Export APIs each allow 100 requests/minute. Reserve headroom and pace requests +// without bursts. Wiki listing has its own budget; other APIs have separate +// reactive cooldowns by API path template. The registry spans clients, data +// sources and task retries in this process. Different App IDs and API hosts +// have independent budgets. +var feishuRequestGates sync.Map + +type feishuRequestGateKey struct { + baseURL string + appID string + api string +} + +type feishuRequestGate struct { + api string + mu sync.Mutex + interval time.Duration + nextRequest time.Time + blockedUntil time.Time +} + +func feishuRequestInterval() time.Duration { + rpm, err := strconv.Atoi(strings.TrimSpace(os.Getenv("FEISHU_API_REQUESTS_PER_MINUTE"))) + if err != nil || rpm <= 0 || int64(rpm) > int64(time.Minute) { + rpm = defaultFeishuRequestsPerMinute + } + return time.Minute / time.Duration(rpm) +} + +func feishuRequestAPI(method, path string) (string, bool) { + path, _, _ = strings.Cut(path, "?") + path = strings.TrimRight(path, "/") + if method == http.MethodPost && path == "/open-apis/drive/v1/export_tasks" { + return "export-create", true + } + if method == http.MethodGet && strings.HasPrefix(path, "/open-apis/drive/v1/export_tasks/") { + if strings.HasPrefix(path, "/open-apis/drive/v1/export_tasks/file/") && strings.HasSuffix(path, "/download") { + return "export-download", true + } + return "export-status", true + } + if method == http.MethodGet && + strings.HasPrefix(path, "/open-apis/wiki/v2/spaces/") && strings.HasSuffix(path, "/nodes") { + return "wiki-node-list", true + } + // Normalize the connector's remaining API paths without retaining document + // tokens, sheet ranges or query parameters. Methods have independent quotas. + parts := strings.Split(path, "/") + for _, template := range []string{ + "/open-apis/drive/v1/files/:token/download", + "/open-apis/drive/v1/medias/:token/download", + "/open-apis/drive/explorer/v2/folder/:token/meta", + "/open-apis/docx/v1/documents/:token/raw_content", + "/open-apis/docx/v1/documents/:token/blocks", + "/open-apis/sheets/v2/spreadsheets/:token/values/:range", + "/open-apis/bitable/v1/apps/:token/tables/:table/fields", + "/open-apis/bitable/v1/apps/:token/tables/:table/records/search", + "/open-apis/wiki/v2/spaces/get_node", + } { + pattern := strings.Split(template, "/") + if len(parts) != len(pattern) { + continue + } + match := true + for i, segment := range pattern { + if !strings.HasPrefix(segment, ":") && segment != parts[i] { + match = false + break + } + } + if match { + return method + " " + template, false + } + } + // Unknown endpoints share only their service/version/resource prefix, never + // a process-wide "other" gate or a gate per document token. + parts = strings.Split(strings.Trim(path, "/"), "/") + return method + " /" + strings.Join(parts[:min(4, len(parts))], "/"), false +} + +func feishuConfiguredDuration(name string, fallback time.Duration) time.Duration { + configured, err := time.ParseDuration(strings.TrimSpace(os.Getenv(name))) + if err == nil && configured > 0 { + return configured + } + return fallback +} + +func feishuExportMaxPolls() int { + configured, err := strconv.Atoi(strings.TrimSpace(os.Getenv("FEISHU_EXPORT_MAX_POLLS"))) + if err == nil && configured > 0 { + return configured + } + return defaultFeishuExportMaxPolls +} + +func (c *Client) requestGate(method, path string) *feishuRequestGate { + api, paced := feishuRequestAPI(method, path) + key := feishuRequestGateKey{strings.TrimRight(c.baseURL, "/"), c.appID, api} + if existing, ok := feishuRequestGates.Load(key); ok { + return existing.(*feishuRequestGate) + } + gate := &feishuRequestGate{api: api} + if paced { + gate.interval = feishuRequestInterval() + } + actual, _ := feishuRequestGates.LoadOrStore(key, gate) + return actual.(*feishuRequestGate) +} + +// wait reserves the next free slot before sleeping, so callers are granted in +// arrival order instead of racing for each slot after a shared wake-up. A +// cooldown imposed while sleeping invalidates the reservation; the caller waits +// it out and reserves again. Cancelled reservations are not reused, which only +// errs towards sending fewer requests. +func (g *feishuRequestGate) wait(ctx context.Context) error { + for { + if err := ctx.Err(); err != nil { + return err + } + g.mu.Lock() + now := time.Now() + if blocked := g.blockedUntil.Sub(now); blocked > 0 { + g.mu.Unlock() + if err := sleepCtx(ctx, blocked); err != nil { + return err + } + continue + } + slot := now + if g.nextRequest.After(slot) { + slot = g.nextRequest + } + g.nextRequest = slot.Add(g.interval) + g.mu.Unlock() + if err := sleepCtx(ctx, slot.Sub(now)); err != nil { + return err + } + g.mu.Lock() + blocked := g.blockedUntil.After(time.Now()) + g.mu.Unlock() + if !blocked { + return nil + } + } +} + +// Even an exhausted retry delays the next document, rather than letting a long +// sync continue hammering an API which has already rejected this application. +func (g *feishuRequestGate) cooldown(wait time.Duration) { + wait = min(wait, feishuConfiguredDuration("FEISHU_RATE_LIMIT_MAX_WAIT", defaultFeishuMaxCooldown)) + g.mu.Lock() + defer g.mu.Unlock() + until := time.Now().Add(wait) + if until.After(g.blockedUntil) { + g.blockedUntil = until + } +} + +func isFeishuRateLimited(status int, body []byte) bool { + if status == http.StatusTooManyRequests { + return true + } + if status != http.StatusBadRequest && status != http.StatusOK { + return false + } + // Successful block/record responses can be large. Only JSON-decode the + // envelope when it could contain the legacy rate-limit code. + if !bytes.Contains(body, feishuRateLimitCodeBytes) { + return false + } + var response struct { + Code int `json:"code"` + } + return json.Unmarshal(body, &response) == nil && response.Code == feishuRateLimitCode +} + +func feishuRateLimitWait(headers http.Header, now time.Time) time.Duration { + var wait time.Duration + valid := false + for _, name := range []string{"x-ogw-ratelimit-reset", "Retry-After"} { + value := strings.TrimSpace(headers.Get(name)) + if value == "" { + continue + } + seconds, err := strconv.ParseFloat(value, 64) + if err != nil && name == "Retry-After" { + if date, dateErr := http.ParseTime(value); dateErr == nil { + seconds, err = date.Sub(now).Seconds(), nil + } + } + if err != nil || math.IsNaN(seconds) || math.IsInf(seconds, 0) || + seconds >= float64(math.MaxInt64)/float64(time.Second) { + continue + } + valid = true + candidate := 100 * time.Millisecond + if seconds > 0 { + candidate = time.Duration(seconds * float64(time.Second)) + } + if candidate > wait { + wait = candidate + } + } + if !valid { + wait = feishuConfiguredDuration("FEISHU_RATE_LIMIT_RETRY_WAIT", defaultFeishuRateLimitWait) + } + return min(wait, feishuConfiguredDuration("FEISHU_RATE_LIMIT_MAX_WAIT", defaultFeishuMaxCooldown)) +} diff --git a/internal/datasource/connector/feishu/core/request_limits_test.go b/internal/datasource/connector/feishu/core/request_limits_test.go new file mode 100644 index 00000000..a2882df3 --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_limits_test.go @@ -0,0 +1,530 @@ +package core + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" +) + +func TestFeishuRequestIntervalConfiguration(t *testing.T) { + for _, tt := range []struct { + value string + want time.Duration + }{ + {"", 750 * time.Millisecond}, + {"80", 750 * time.Millisecond}, + {"60", time.Second}, + {"0", 750 * time.Millisecond}, + {"-1", 750 * time.Millisecond}, + {"invalid", 750 * time.Millisecond}, + } { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", tt.value) + if got := feishuRequestInterval(); got != tt.want { + t.Errorf("interval(%q)=%s, want %s", tt.value, got, tt.want) + } + } + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "7s") + if got := feishuRateLimitWait(http.Header{}, time.Now()); got != 7*time.Second { + t.Fatalf("configured fallback = %s", got) + } +} + +func TestFeishuGatesShareAppAndAPIIndependentOfDocument(t *testing.T) { + resetFeishuRequestGates(t) + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "80") + first := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test"}) + second := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test/"}) + poll := first.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/ticket-one?token=one") + if poll != second.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/ticket-two?token=two") { + t.Fatal("ticket and query must not create independent rate budgets") + } + if poll.interval != 750*time.Millisecond { + t.Fatalf("default interval = %s", poll.interval) + } + create := first.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") + download := first.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/file/one/download") + if create == poll || create == download || poll == download { + t.Fatal("export APIs must have separate budgets") + } + if download != second.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/file/two/download") { + t.Fatal("download tokens must share one budget") + } + otherApp := NewClient(&Config{AppID: t.Name() + "-other", BaseURL: first.baseURL}) + otherHost := NewClient(&Config{AppID: t.Name(), BaseURL: "http://other-host"}) + if create == otherApp.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") || + create == otherHost.requestGate(http.MethodPost, "/open-apis/drive/v1/export_tasks") { + t.Fatal("different applications and API hosts must not share a budget") + } + nodes := first.requestGate(http.MethodGet, "/open-apis/wiki/v2/spaces/one/nodes?parent_node_token=one") + if nodes != second.requestGate(http.MethodGet, "/open-apis/wiki/v2/spaces/two/nodes?parent_node_token=two") || + nodes.interval != poll.interval { + t.Fatal("wiki list paths must share a paced budget") + } +} + +func TestFeishuGatePacesConcurrentClients(t *testing.T) { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "3000") // 20ms for a bounded test. + var mu sync.Mutex + var arrivals []time.Time + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + arrivals = append(arrivals, time.Now()) + mu.Unlock() + writeJSON(w, ApiResponse{Code: 0}) + }) + defer ts.Close() + clients := []*Client{NewClient(cfg), NewClient(cfg)} + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + err := clients[i%2].DoRequest(context.Background(), + http.MethodPost, "/open-apis/drive/v1/export_tasks", nil, nil) + if err != nil { + t.Errorf("request: %v", err) + } + }(i) + } + wg.Wait() + sort.Slice(arrivals, func(i, j int) bool { return arrivals[i].Before(arrivals[j]) }) + if len(arrivals) != 6 { + t.Fatalf("received %d requests", len(arrivals)) + } + if elapsed := arrivals[5].Sub(arrivals[0]); elapsed < 90*time.Millisecond { + t.Fatalf("clients burst through shared budget: elapsed=%s", elapsed) + } +} + +func TestFeishuLegacyRateLimitRetriesJSONAndDownload(t *testing.T) { + for _, tt := range []struct { + name string + download bool + status int + }{ + {"json_400", false, http.StatusBadRequest}, + {"json_200", false, http.StatusOK}, + {"download_400", true, http.StatusBadRequest}, + } { + t.Run(tt.name, func(t *testing.T) { + var attempts atomic.Int32 + var firstAttempt atomic.Int64 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) == 1 { + firstAttempt.Store(time.Now().UnixNano()) + w.Header().Set("x-ogw-ratelimit-reset", "0.05") + w.WriteHeader(tt.status) + _, _ = io.WriteString(w, `{"code":99991400,"msg":"request trigger frequency limit"}`) + return + } + if time.Since(time.Unix(0, firstAttempt.Load())) < 50*time.Millisecond { + t.Error("request retried before Feishu reset time") + } + if tt.download { + _, _ = io.WriteString(w, "file-content") + } else { + writeJSON(w, ApiResponse{Code: 0}) + } + }) + defer ts.Close() + c := NewClient(cfg) + if tt.download { + data, err := c.downloadRawBytes(context.Background(), "/target") + if err != nil || string(data) != "file-content" { + t.Fatalf("download=%q err=%v", data, err) + } + } else if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err != nil { + t.Fatal(err) + } + if attempts.Load() != 2 { + t.Fatalf("attempts=%d, want 2", attempts.Load()) + } + }) + } +} + +func TestFeishuCooldownBlocksOtherClientAndHonorsCancellation(t *testing.T) { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.Header().Set("x-ogw-ratelimit-reset", "0.2") + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":99991400}`) + }) + defer ts.Close() + first := NewClient(cfg) + ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) + defer cancel() + if err := first.DoRequest(ctx, http.MethodGet, "/target", nil, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wait should be cancellable: %v", err) + } + second := NewClient(cfg) + ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel2() + if err := second.DoRequest(ctx2, http.MethodGet, "/target", nil, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("other client must inherit cooldown: %v", err) + } + if got := attempts.Load(); got != 1 { + t.Fatalf("cooldown allowed extra requests: %d", got) + } +} + +func TestFeishuRateLimitExhaustionProtectsNextDocument(t *testing.T) { + var attempts atomic.Int32 + ts, cfg := retryTestServer(t, "/target", func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.Header().Set("Retry-After", "0.05") + w.WriteHeader(http.StatusTooManyRequests) + }) + defer ts.Close() + c := NewClient(cfg) + if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { + t.Fatal("persistent rate limit must not report success") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + err := NewClient(cfg).DoRequest(ctx, http.MethodGet, "/target", nil, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("next document must wait after exhausted retries: %v", err) + } + if got := attempts.Load(); got != 4 { + t.Fatalf("attempts=%d, want bounded 4", got) + } +} + +// Gate tests are serial (no t.Parallel): clear before and after each test so +// even a reused httptest port cannot inherit pacing or cooldown state. +func resetFeishuRequestGates(t *testing.T) { + t.Helper() + feishuRequestGates.Clear() + t.Cleanup(feishuRequestGates.Clear) +} + +func TestFeishuRequestGateReset(t *testing.T) { + config := &Config{AppID: t.Name(), BaseURL: "http://reused-test-host"} + path := "/open-apis/drive/v1/export_tasks" + old := NewClient(config).requestGate(http.MethodPost, path) + old.cooldown(time.Minute) + t.Run("reset", func(t *testing.T) { + resetFeishuRequestGates(t) + fresh := NewClient(config).requestGate(http.MethodPost, path) + if fresh == old || !fresh.blockedUntil.IsZero() || !fresh.nextRequest.IsZero() { + t.Fatal("reused host/app/API inherited gate state") + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := fresh.wait(ctx); err != nil { + t.Fatalf("fresh gate still blocked: %v", err) + } + }) + key := feishuRequestGateKey{config.BaseURL, config.AppID, "export-create"} + if _, exists := feishuRequestGates.Load(key); exists { + t.Fatal("test cleanup left a gate in the registry") + } +} + +func TestFeishuCooldownConfigurationAndBounds(t *testing.T) { + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "25ms") + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "1h") + if got := feishuRateLimitWait(http.Header{}, time.Now()); got != 25*time.Millisecond { + t.Fatalf("fallback escaped cap: %s", got) + } + headers := http.Header{"Retry-After": {"86400"}} + if got := feishuRateLimitWait(headers, time.Now()); got != 25*time.Millisecond { + t.Fatalf("header escaped configured cap: %s", got) + } + headers.Set("Retry-After", time.Now().Add(24*time.Hour).UTC().Format(http.TimeFormat)) + if got := feishuRateLimitWait(headers, time.Now()); got != 25*time.Millisecond { + t.Fatalf("HTTP date escaped configured cap: %s", got) + } + gate := &feishuRequestGate{} + gate.cooldown(time.Hour) + // Check the remaining cooldown, excluding any time the test was descheduled + // before cooldown() ran. Scheduler latency must not look like an excessive cap. + if remaining := time.Until(gate.blockedUntil); remaining > 25*time.Millisecond { + t.Fatalf("shared gate escaped cap: %s", remaining) + } + for _, invalid := range []string{"", "0s", "-1s", "bad"} { + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", invalid) + if got := feishuRateLimitWait(headers, time.Now()); got != 5*time.Minute { + t.Fatalf("invalid cap %q: got %s", invalid, got) + } + } +} + +func BenchmarkFeishuRateLimitSuccess(b *testing.B) { + body := append([]byte(`{"code":0,"data":"`), bytes.Repeat([]byte("x"), 16<<20)...) + body = append(body, []byte(`"}`)...) + b.SetBytes(int64(len(body))) + b.ResetTimer() + for b.Loop() { + if isFeishuRateLimited(http.StatusOK, body) { + b.Fatal("successful response classified as a rate limit") + } + } +} + +func TestFeishuReactiveCooldownsArePerAPI(t *testing.T) { + resetFeishuRequestGates(t) + c := NewClient(&Config{AppID: t.Name(), BaseURL: "http://limits-test"}) + pairs := [][2]string{ + {"/open-apis/drive/v1/files/one/download", "/open-apis/drive/v1/files/two/download?x=y"}, + {"/open-apis/drive/v1/medias/one/download", "/open-apis/drive/v1/medias/two/download"}, + {"/open-apis/docx/v1/documents/one/blocks", "/open-apis/docx/v1/documents/two/blocks?page_token=x"}, + {"/open-apis/docx/v1/documents/one/raw_content", "/open-apis/docx/v1/documents/two/raw_content"}, + {"/open-apis/sheets/v2/spreadsheets/one/values/A1", "/open-apis/sheets/v2/spreadsheets/two/values/B2"}, + {"/open-apis/bitable/v1/apps/one/tables/t1/fields", "/open-apis/bitable/v1/apps/two/tables/t2/fields"}, + { + "/open-apis/bitable/v1/apps/one/tables/t1/records/search", + "/open-apis/bitable/v1/apps/two/tables/t2/records/search", + }, + {"/open-apis/drive/explorer/v2/folder/one/meta", "/open-apis/drive/explorer/v2/folder/two/meta"}, + {"/open-apis/wiki/v2/spaces/get_node?token=one", "/open-apis/wiki/v2/spaces/get_node?token=two"}, + {"/open-apis/wiki/v2/spaces?page_token=one", "/open-apis/wiki/v2/spaces?page_token=two"}, + } + seen := make(map[*feishuRequestGate]bool) + for _, pair := range pairs { + gate := c.requestGate(http.MethodGet, pair[0]) + if gate != c.requestGate(http.MethodGet, pair[1]) || gate.interval != 0 { + t.Fatalf("reactive gate must normalize tokens and query: %v", pair) + } + if seen[gate] { + t.Fatalf("unrelated APIs share a cooldown: %v", pair) + } + seen[gate] = true + if gate == c.requestGate(http.MethodPost, pair[0]) { + t.Fatalf("HTTP methods share a cooldown: %v", pair) + } + } + media := c.requestGate(http.MethodGet, pairs[1][0]) + media.cooldown(time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := c.requestGate(http.MethodGet, pairs[2][0]).wait(ctx); err != nil { + t.Fatalf("media cooldown blocked docx: %v", err) + } + if err := media.wait(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("media did not retain its own cooldown: %v", err) + } +} + +func TestFeishuRateLimitDetection(t *testing.T) { + for _, tt := range []struct { + status int + body string + want bool + }{ + {http.StatusOK, `{"code":0,"data":{"text":"99991400"}}`, false}, + {http.StatusOK, `{"code":99991400}`, true}, + {http.StatusBadRequest, `{"code":99991400}`, true}, + {http.StatusOK, `{"code":0}`, false}, + {http.StatusOK, `invalid 99991400`, false}, + {http.StatusForbidden, `{"code":99991400}`, false}, + {http.StatusTooManyRequests, ``, true}, + } { + if got := isFeishuRateLimited(tt.status, []byte(tt.body)); got != tt.want { + t.Errorf("rate limited(%d, %s) = %v, want %v", tt.status, tt.body, got, tt.want) + } + } +} + +// Run HTTP handlers in process so synctest can advance time without real sockets. +type feishuHandlerTransport struct { + http.Handler +} + +func (tr feishuHandlerTransport) RoundTrip(r *http.Request) (*http.Response, error) { + recorder := httptest.NewRecorder() + tr.ServeHTTP(recorder, r) + return recorder.Result(), nil +} + +func TestFeishuExportPollingRetainsTicketAcrossCooldown(t *testing.T) { + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "80") + t.Setenv("FEISHU_RATE_LIMIT_RETRY_WAIT", "70s") + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "5m") + t.Setenv("FEISHU_EXPORT_MAX_POLLS", "2") + t.Setenv("FEISHU_EXPORT_POLL_INTERVAL", "2s") + synctest.Test(t, func(t *testing.T) { + resetFeishuRequestGates(t) + var creates, polls, downloads int + mux := http.NewServeMux() + mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, _ *http.Request) { + creates++ + _, _ = io.WriteString(w, `{"code":0,"data":{"ticket":"same-ticket"}}`) + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/same-ticket", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("token") != "document" { + t.Error("poll did not retain the document token") + } + polls++ + switch polls { + case 1: + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":99991400}`) + case 2: + _, _ = io.WriteString(w, `{"code":0,"data":{"result":{"job_status":2}}}`) + default: + _, _ = io.WriteString(w, + `{"code":0,"data":{"result":{"job_status":0,"file_token":"file","file_name":"doc.pdf"}}}`) + } + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/file/download", + func(w http.ResponseWriter, _ *http.Request) { + downloads++ + _, _ = io.WriteString(w, "export-content") + }) + cfg := &Config{AppID: t.Name(), BaseURL: "http://feishu-export.test"} + c := NewClient(cfg) + c.httpClient.Transport = feishuHandlerTransport{mux} + c.tokenCache, c.tokenExpAt = "cached-token", time.Now().Add(time.Hour) + // Both shared and in-poll cooldowns exceed the previous 60-second deadline. + started := time.Now() + other := NewClient(cfg) + other.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/other-ticket").cooldown(70 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + data, name, err := c.ExportAndDownload(ctx, "document", "docx") + if err != nil || name != "doc.pdf" || string(data) != "export-content" { + t.Fatalf("export across cooldown: data=%q name=%q err=%v", data, name, err) + } + if creates != 1 || polls != 3 || downloads != 1 { + t.Fatalf("create/poll/download=%d/%d/%d, want 1/3/1", creates, polls, downloads) + } + if elapsed := time.Since(started); elapsed < 140*time.Second { + t.Fatalf("export skipped the shared or in-poll cooldown: %s", elapsed) + } + }) +} + +func TestFeishuExportPollingIsBoundedAndCancellable(t *testing.T) { + for _, cancelWhileWaiting := range []bool{false, true} { + t.Run(fmt.Sprint(cancelWhileWaiting), func(t *testing.T) { + t.Setenv("FEISHU_EXPORT_MAX_POLLS", "3") + t.Setenv("FEISHU_EXPORT_POLL_INTERVAL", "1ms") + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "60000") + var creates, polls atomic.Int32 + createHandler := func(w http.ResponseWriter, _ *http.Request) { + creates.Add(1) + _, _ = io.WriteString(w, `{"code":0,"data":{"ticket":"pending"}}`) + } + ts, cfg := retryTestServer(t, "/open-apis/drive/v1/export_tasks", createHandler) + defer ts.Close() + ts.Config.Handler.(*http.ServeMux).HandleFunc("/open-apis/drive/v1/export_tasks/pending", + func(w http.ResponseWriter, _ *http.Request) { + polls.Add(1) + _, _ = io.WriteString(w, `{"code":0,"data":{"result":{"job_status":2}}}`) + }) + c := NewClient(cfg) + if cancelWhileWaiting { + c.requestGate(http.MethodGet, "/open-apis/drive/v1/export_tasks/pending").cooldown(time.Second) + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, _, err := c.ExportAndDownload(ctx, "document", "docx") + if cancelWhileWaiting { + if !errors.Is(err, context.DeadlineExceeded) || polls.Load() != 0 { + t.Fatalf("polling ignored cancellation: polls=%d err=%v", polls.Load(), err) + } + } else if err == nil || !strings.Contains(err.Error(), "after 3 polls") || polls.Load() != 3 { + t.Fatalf("polling is unbounded: polls=%d err=%v", polls.Load(), err) + } + // Exercise the same wrapping and metadata path used by the connectors. + metadata := FeishuErrorItemMeta(fmt.Errorf("export document: %w", err), nil) + if metadata["error_reason_code"] != "feishu_timeout" { + t.Fatalf("polling lost its timeout classification: %v", metadata) + } + if strings.Contains(metadata["error_reason"], "ticket=") { + t.Fatal("export ticket leaked into the user-facing failure reason") + } + if creates.Load() != 1 { + t.Fatalf("created %d tasks, want 1", creates.Load()) + } + }) + } +} + +func TestFeishuRefreshesTokenAfterCooldown(t *testing.T) { + for _, download := range []bool{false, true} { + t.Run(fmt.Sprint(download), func(t *testing.T) { + resetFeishuRequestGates(t) + var auths, attempts atomic.Int32 + var c *Client + mux := http.NewServeMux() + authHandler := func(w http.ResponseWriter, _ *http.Request) { + token := fmt.Sprintf("token-%d", auths.Add(1)) + writeJSON(w, TokenResponse{TenantAccessToken: token, Expire: 7200}) + } + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", authHandler) + mux.HandleFunc("/target", func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) == 1 { + c.tokenMu.Lock() + c.tokenExpAt = time.Now().Add(5 * time.Millisecond) + c.tokenMu.Unlock() + w.Header().Set("Retry-After", "0.03") + w.WriteHeader(http.StatusTooManyRequests) + return + } + if r.Header.Get("Authorization") != "Bearer token-2" { + t.Error("request reused the token that expired during cooldown") + } + _, _ = io.WriteString(w, `{"code":0}`) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + c = NewClient(&Config{AppID: t.Name(), BaseURL: ts.URL}) + var err error + if download { + _, err = c.downloadRawBytes(context.Background(), "/target") + } else { + err = c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil) + } + if err != nil || auths.Load() != 2 || attempts.Load() != 2 { + t.Fatalf("auth/attempts=%d/%d err=%v", auths.Load(), attempts.Load(), err) + } + }) + } +} + +func TestFeishuGateGrantsInArrivalOrder(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + gate := &feishuRequestGate{interval: time.Second} + start := time.Now() + var mu sync.Mutex + var order []int + var grants []time.Duration + var wg sync.WaitGroup + for i := range 5 { + wg.Add(1) + go func() { + defer wg.Done() + if err := gate.wait(context.Background()); err != nil { + t.Error(err) + return + } + mu.Lock() + order = append(order, i) + grants = append(grants, time.Since(start)) + mu.Unlock() + }() + synctest.Wait() + } + wg.Wait() + for i := range 5 { + if order[i] != i || grants[i] != time.Duration(i)*time.Second { + t.Fatalf("grants out of arrival order or pace: order=%v grants=%v", order, grants) + } + } + }) +} diff --git a/internal/datasource/connector/feishu/core/request_token_test.go b/internal/datasource/connector/feishu/core/request_token_test.go new file mode 100644 index 00000000..690ac54a --- /dev/null +++ b/internal/datasource/connector/feishu/core/request_token_test.go @@ -0,0 +1,121 @@ +package core + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sort" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Cached-token requests first check Err when entering gate.wait. Observe that +// boundary so the test expires the token only after every caller authenticated. +type feishuGateWaitContext struct { + context.Context + once sync.Once + entered chan<- struct{} +} + +func (c *feishuGateWaitContext) Err() error { + c.once.Do(func() { c.entered <- struct{}{} }) + return c.Context.Err() +} + +func TestFeishuTokenRefreshPreservesPacingAndCooldown(t *testing.T) { + for _, mode := range []string{"json", "download"} { + t.Run(mode, func(t *testing.T) { + resetFeishuRequestGates(t) + t.Setenv("FEISHU_API_REQUESTS_PER_MINUTE", "600") + t.Setenv("FEISHU_RATE_LIMIT_MAX_WAIT", "5m") + const count = 4 + const interval = 100 * time.Millisecond + var mu sync.Mutex + var arrivals []time.Time + var earliestRequest time.Time + var auths atomic.Int32 + var gate *feishuRequestGate + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" { + auths.Add(1) + // Slow enough for every caller to pass the gate in the broken + // implementation and queue behind the refreshing caller's tokenMu. + time.Sleep(time.Duration(count+1) * interval) + mu.Lock() + earliestRequest = time.Now().Add(3 * interval) + mu.Unlock() + // A concurrent data source can extend this shared cooldown while + // our token is refreshing. All callers must honor the new delay. + gate.cooldown(3 * interval) + writeJSON(w, TokenResponse{TenantAccessToken: "fresh-token", Expire: 7200}) + return + } + if r.Header.Get("Authorization") != "Bearer fresh-token" { + t.Error("request sent the token that expired while waiting") + } + mu.Lock() + arrivals = append(arrivals, time.Now()) + mu.Unlock() + _, _ = io.WriteString(w, `{"code":0}`) + })) + defer server.Close() + c := NewClient(&Config{AppID: t.Name(), BaseURL: server.URL}) + c.tokenCache = "cached-token" + c.tokenExpAt = time.Now().Add(time.Hour) + method, path := http.MethodPost, "/open-apis/drive/v1/export_tasks" + if mode == "download" { + method, path = http.MethodGet, "/open-apis/drive/v1/export_tasks/file/export-token/download" + } + gate = c.requestGate(method, path) + gate.cooldown(500 * time.Millisecond) + entered := make(chan struct{}, count) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var wg sync.WaitGroup + for range count { + wg.Add(1) + go func() { + defer wg.Done() + observed := &feishuGateWaitContext{Context: ctx, entered: entered} + var err error + if mode == "download" { + _, err = c.downloadRawBytes(observed, path) + } else { + err = c.DoRequest(observed, method, path, nil, nil) + } + if err != nil { + t.Errorf("request: %v", err) + } + }() + } + for range count { + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("requests failed to reach the gate") + } + } + c.tokenMu.Lock() + c.tokenExpAt = time.Now().Add(-time.Second) + c.tokenMu.Unlock() + wg.Wait() + mu.Lock() + defer mu.Unlock() + if len(arrivals) != count || auths.Load() != 1 { + t.Fatalf("requests/refreshes=%d/%d, want %d/1", len(arrivals), auths.Load(), count) + } + sort.Slice(arrivals, func(i, j int) bool { return arrivals[i].Before(arrivals[j]) }) + if arrivals[0].Before(earliestRequest) { + t.Errorf("request ignored the cooldown imposed during token refresh") + } + for i := 1; i < len(arrivals); i++ { + if gap := arrivals[i].Sub(arrivals[i-1]); gap < interval/2 { + t.Errorf("requests burst after token refresh: gap=%s, interval=%s", gap, interval) + } + } + }) + } +} diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.go index fb6cbb00..eec1cce5 100644 --- a/internal/datasource/connector/feishu/core/shared.go +++ b/internal/datasource/connector/feishu/core/shared.go @@ -3,6 +3,7 @@ package core import ( "context" "encoding/json" + "errors" "fmt" "maps" "net/http" @@ -105,7 +106,8 @@ func feishuFailure(err error) (code, codeValue, fallback string) { return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes" case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"): return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync" - case strings.Contains(s, "timed out"), + case errors.Is(err, errExportPollsExhausted), + strings.Contains(s, "timed out"), strings.Contains(s, "timeout"), strings.Contains(s, "deadline exceeded"): return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"