fix(models): keep Retry-After on HTTPError and honour it in agent LLM retries

Endpoint.Do turns a non-2xx reply into &HTTPError{StatusCode, Body} and
drops the response headers. A caller that retries a 429 or 503 therefore
cannot see the provider's Retry-After and falls back to its own guess: the
agent retry loop waits 1s, 2s, 3s, which against a provider asking for, say,
20s spends every attempt inside the rate-limit window and then fails the
turn.

- HTTPError carries the reply's Header, and RetryAfter() parses the header
  in both forms (delay-seconds and HTTP-date); absent, malformed or past
  values give zero.
- The agent retry loop waits max(linear backoff, Retry-After), with the
  vendor's value capped at 30s so a large Retry-After cannot hold a turn
  indefinitely.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
main
Victor Maznyuk 2026-09-26 09:04:17 +02:00 committed by lyingbug
parent 15d4845521
commit cf079b8dcd
5 changed files with 154 additions and 2 deletions

View File

@ -145,6 +145,27 @@ func isTransientError(err error) bool {
return false
}
// maxLLMRetryAfter caps how long a vendor's Retry-After may hold a turn
// between two attempts. A longer wait is not worth keeping the user on a
// silent turn for; the retry then goes out at the cap and, if the vendor is
// still limiting, the turn fails the usual way.
const maxLLMRetryAfter = 30 * time.Second
// llmRetryDelay is the wait before retry number attempt (1-based). It is the
// linear backoff, raised to the vendor's Retry-After when a 429/503 reply
// carried one, so a rate-limited request is not re-sent before the provider
// said it would accept it.
func llmRetryDelay(err error, attempt int) time.Duration {
delay := time.Duration(attempt) * time.Second
var httpErr *api.HTTPError
if errors.As(err, &httpErr) {
if wait := min(httpErr.RetryAfter(), maxLLMRetryAfter); wait > delay {
delay = wait
}
}
return delay
}
// getLLMStallTimeout returns how long an LLM stream may go silent before it is
// cancelled, from AgentConfig.LLMCallTimeout or the default.
func (e *AgentEngine) getLLMStallTimeout() time.Duration {

View File

@ -0,0 +1,38 @@
package agent
import (
"errors"
"fmt"
"net/http"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/stretchr/testify/require"
)
func TestLLMRetryDelayHonoursRetryAfter(t *testing.T) {
limited := func(retryAfter string) error {
return fmt.Errorf("chat: %w", &api.HTTPError{
StatusCode: http.StatusTooManyRequests,
Header: http.Header{"Retry-After": []string{retryAfter}},
})
}
for _, tc := range []struct {
name string
err error
attempt int
want time.Duration
}{
{"no header keeps the linear backoff", &api.HTTPError{StatusCode: 503}, 2, 2 * time.Second},
{"untyped error keeps the linear backoff", errors.New("timeout"), 1, time.Second},
{"retry-after longer than the backoff wins", limited("7"), 1, 7 * time.Second},
{"retry-after shorter than the backoff is not a speed-up", limited("1"), 3, 3 * time.Second},
{"retry-after is capped", limited("3600"), 1, maxLLMRetryAfter},
{"malformed retry-after is ignored", limited("soon"), 1, time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, llmRetryDelay(tc.err, tc.attempt))
})
}
}

View File

@ -571,7 +571,7 @@ func (e *AgentEngine) callLLMWithRetry(
if err != nil && isTransientError(err) {
// Retry transient errors (timeout, rate limit, server errors) up to maxLLMRetries times
for retry := 1; retry <= maxLLMRetries; retry++ {
retryDelay := time.Duration(retry) * time.Second
retryDelay := llmRetryDelay(err, retry)
logger.Warnf(ctx, "[Agent][Round-%d] LLM transient error (attempt %d/%d), retrying in %v: %v",
round, retry, maxLLMRetries, retryDelay, err)
// A stop pressed during the backoff ends the turn now rather than

View File

@ -7,7 +7,9 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
@ -139,7 +141,7 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
_ = resp.Body.Close()
return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body)}
return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body), Header: resp.Header}
}
return resp, nil
}
@ -201,12 +203,45 @@ func (e *TransportError) Unwrap() error { return e.Err }
type HTTPError struct {
StatusCode int
Body string
// Header is the reply's headers, kept so a retry can honour the vendor's
// Retry-After on a 429 or 503 instead of guessing its own backoff.
Header http.Header
}
func (e *HTTPError) Error() string {
return fmt.Sprintf("API request failed with status %d: %s", e.StatusCode, e.Body)
}
// RetryAfter is how long the vendor asked the caller to wait before trying
// again, from the Retry-After header in either of its two forms (delay in
// seconds or an HTTP date). It is zero when the header is absent, malformed
// or already in the past.
func (e *HTTPError) RetryAfter() time.Duration {
if e == nil {
return 0
}
return parseRetryAfter(e.Header.Get("Retry-After"), time.Now())
}
func parseRetryAfter(value string, now time.Time) time.Duration {
value = strings.TrimSpace(value)
if value == "" {
return 0
}
if secs, err := strconv.Atoi(value); err == nil {
if secs <= 0 {
return 0
}
return time.Duration(secs) * time.Second
}
if at, err := http.ParseTime(value); err == nil {
if d := at.Sub(now); d > 0 {
return d
}
}
return 0
}
// LogRequest emits the standard request log line with image payloads
// compacted so data URIs do not flood the log.
func LogRequest(ctx context.Context, url, model string, body []byte, stream bool) {

View File

@ -0,0 +1,58 @@
package api
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestDoKeepsHeadersOnHTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "12")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = w.Write([]byte(`{"error":"rate limited"}`))
}))
defer srv.Close()
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, srv.URL, nil)
require.NoError(t, err)
_, err = Endpoint{Client: srv.Client()}.Do(req)
var httpErr *HTTPError
require.True(t, errors.As(err, &httpErr))
require.Equal(t, http.StatusTooManyRequests, httpErr.StatusCode)
require.Equal(t, 12*time.Second, httpErr.RetryAfter())
}
func TestParseRetryAfter(t *testing.T) {
now := time.Date(2026, 9, 26, 12, 0, 0, 0, time.UTC)
for _, tc := range []struct {
name string
value string
want time.Duration
}{
{"absent", "", 0},
{"seconds", "30", 30 * time.Second},
{"seconds with spaces", " 5 ", 5 * time.Second},
{"zero", "0", 0},
{"negative", "-3", 0},
{"http date", now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second},
{"http date in the past", now.Add(-time.Minute).Format(http.TimeFormat), 0},
{"garbage", "tomorrow", 0},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, parseRetryAfter(tc.value, now))
})
}
}
func TestRetryAfterOnNilError(t *testing.T) {
var e *HTTPError
require.Zero(t, e.RetryAfter())
require.Zero(t, (&HTTPError{StatusCode: 503}).RetryAfter())
}