diff --git a/internal/agent/const.go b/internal/agent/const.go index cffb61c0..fb6104df 100644 --- a/internal/agent/const.go +++ b/internal/agent/const.go @@ -145,6 +145,27 @@ func isTransientError(err error) bool { return false } +// maxLLMRetryAfter caps how long a vendor's Retry-After may hold a turn +// between two attempts. A longer wait is not worth keeping the user on a +// silent turn for; the retry then goes out at the cap and, if the vendor is +// still limiting, the turn fails the usual way. +const maxLLMRetryAfter = 30 * time.Second + +// llmRetryDelay is the wait before retry number attempt (1-based). It is the +// linear backoff, raised to the vendor's Retry-After when a 429/503 reply +// carried one, so a rate-limited request is not re-sent before the provider +// said it would accept it. +func llmRetryDelay(err error, attempt int) time.Duration { + delay := time.Duration(attempt) * time.Second + var httpErr *api.HTTPError + if errors.As(err, &httpErr) { + if wait := min(httpErr.RetryAfter(), maxLLMRetryAfter); wait > delay { + delay = wait + } + } + return delay +} + // getLLMStallTimeout returns how long an LLM stream may go silent before it is // cancelled, from AgentConfig.LLMCallTimeout or the default. func (e *AgentEngine) getLLMStallTimeout() time.Duration { diff --git a/internal/agent/retry_delay_test.go b/internal/agent/retry_delay_test.go new file mode 100644 index 00000000..f5322ccf --- /dev/null +++ b/internal/agent/retry_delay_test.go @@ -0,0 +1,38 @@ +package agent + +import ( + "errors" + "fmt" + "net/http" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/stretchr/testify/require" +) + +func TestLLMRetryDelayHonoursRetryAfter(t *testing.T) { + limited := func(retryAfter string) error { + return fmt.Errorf("chat: %w", &api.HTTPError{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": []string{retryAfter}}, + }) + } + for _, tc := range []struct { + name string + err error + attempt int + want time.Duration + }{ + {"no header keeps the linear backoff", &api.HTTPError{StatusCode: 503}, 2, 2 * time.Second}, + {"untyped error keeps the linear backoff", errors.New("timeout"), 1, time.Second}, + {"retry-after longer than the backoff wins", limited("7"), 1, 7 * time.Second}, + {"retry-after shorter than the backoff is not a speed-up", limited("1"), 3, 3 * time.Second}, + {"retry-after is capped", limited("3600"), 1, maxLLMRetryAfter}, + {"malformed retry-after is ignored", limited("soon"), 1, time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, llmRetryDelay(tc.err, tc.attempt)) + }) + } +} diff --git a/internal/agent/think.go b/internal/agent/think.go index c8b88952..e7a5ad10 100644 --- a/internal/agent/think.go +++ b/internal/agent/think.go @@ -571,7 +571,7 @@ func (e *AgentEngine) callLLMWithRetry( if err != nil && isTransientError(err) { // Retry transient errors (timeout, rate limit, server errors) up to maxLLMRetries times for retry := 1; retry <= maxLLMRetries; retry++ { - retryDelay := time.Duration(retry) * time.Second + retryDelay := llmRetryDelay(err, retry) logger.Warnf(ctx, "[Agent][Round-%d] LLM transient error (attempt %d/%d), retrying in %v: %v", round, retry, maxLLMRetries, retryDelay, err) // A stop pressed during the backoff ends the turn now rather than diff --git a/internal/models/api/endpoint.go b/internal/models/api/endpoint.go index 5075753a..3df928d0 100644 --- a/internal/models/api/endpoint.go +++ b/internal/models/api/endpoint.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" + "time" "github.com/Tencent/WeKnora/internal/logger" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -139,7 +141,7 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) { if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) _ = resp.Body.Close() - return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body)} + return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body), Header: resp.Header} } return resp, nil } @@ -201,12 +203,45 @@ func (e *TransportError) Unwrap() error { return e.Err } type HTTPError struct { StatusCode int Body string + // Header is the reply's headers, kept so a retry can honour the vendor's + // Retry-After on a 429 or 503 instead of guessing its own backoff. + Header http.Header } func (e *HTTPError) Error() string { return fmt.Sprintf("API request failed with status %d: %s", e.StatusCode, e.Body) } +// RetryAfter is how long the vendor asked the caller to wait before trying +// again, from the Retry-After header in either of its two forms (delay in +// seconds or an HTTP date). It is zero when the header is absent, malformed +// or already in the past. +func (e *HTTPError) RetryAfter() time.Duration { + if e == nil { + return 0 + } + return parseRetryAfter(e.Header.Get("Retry-After"), time.Now()) +} + +func parseRetryAfter(value string, now time.Time) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if secs, err := strconv.Atoi(value); err == nil { + if secs <= 0 { + return 0 + } + return time.Duration(secs) * time.Second + } + if at, err := http.ParseTime(value); err == nil { + if d := at.Sub(now); d > 0 { + return d + } + } + return 0 +} + // LogRequest emits the standard request log line with image payloads // compacted so data URIs do not flood the log. func LogRequest(ctx context.Context, url, model string, body []byte, stream bool) { diff --git a/internal/models/api/endpoint_test.go b/internal/models/api/endpoint_test.go new file mode 100644 index 00000000..284380b6 --- /dev/null +++ b/internal/models/api/endpoint_test.go @@ -0,0 +1,58 @@ +package api + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDoKeepsHeadersOnHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "12") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate limited"}`)) + })) + defer srv.Close() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, srv.URL, nil) + require.NoError(t, err) + _, err = Endpoint{Client: srv.Client()}.Do(req) + + var httpErr *HTTPError + require.True(t, errors.As(err, &httpErr)) + require.Equal(t, http.StatusTooManyRequests, httpErr.StatusCode) + require.Equal(t, 12*time.Second, httpErr.RetryAfter()) +} + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 9, 26, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + value string + want time.Duration + }{ + {"absent", "", 0}, + {"seconds", "30", 30 * time.Second}, + {"seconds with spaces", " 5 ", 5 * time.Second}, + {"zero", "0", 0}, + {"negative", "-3", 0}, + {"http date", now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second}, + {"http date in the past", now.Add(-time.Minute).Format(http.TimeFormat), 0}, + {"garbage", "tomorrow", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, parseRetryAfter(tc.value, now)) + }) + } +} + +func TestRetryAfterOnNilError(t *testing.T) { + var e *HTTPError + require.Zero(t, e.RetryAfter()) + require.Zero(t, (&HTTPError{StatusCode: 503}).RetryAfter()) +}