fix(models): keep Retry-After on HTTPError and honour it in agent LLM retries
Endpoint.Do turns a non-2xx reply into &HTTPError{StatusCode, Body} and
drops the response headers. A caller that retries a 429 or 503 therefore
cannot see the provider's Retry-After and falls back to its own guess: the
agent retry loop waits 1s, 2s, 3s, which against a provider asking for, say,
20s spends every attempt inside the rate-limit window and then fails the
turn.
- HTTPError carries the reply's Header, and RetryAfter() parses the header
in both forms (delay-seconds and HTTP-date); absent, malformed or past
values give zero.
- The agent retry loop waits max(linear backoff, Retry-After), with the
vendor's value capped at 30s so a large Retry-After cannot hold a turn
indefinitely.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
main
parent
15d4845521
commit
cf079b8dcd
|
|
@ -145,6 +145,27 @@ func isTransientError(err error) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// maxLLMRetryAfter caps how long a vendor's Retry-After may hold a turn
|
||||
// between two attempts. A longer wait is not worth keeping the user on a
|
||||
// silent turn for; the retry then goes out at the cap and, if the vendor is
|
||||
// still limiting, the turn fails the usual way.
|
||||
const maxLLMRetryAfter = 30 * time.Second
|
||||
|
||||
// llmRetryDelay is the wait before retry number attempt (1-based). It is the
|
||||
// linear backoff, raised to the vendor's Retry-After when a 429/503 reply
|
||||
// carried one, so a rate-limited request is not re-sent before the provider
|
||||
// said it would accept it.
|
||||
func llmRetryDelay(err error, attempt int) time.Duration {
|
||||
delay := time.Duration(attempt) * time.Second
|
||||
var httpErr *api.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
if wait := min(httpErr.RetryAfter(), maxLLMRetryAfter); wait > delay {
|
||||
delay = wait
|
||||
}
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
// getLLMStallTimeout returns how long an LLM stream may go silent before it is
|
||||
// cancelled, from AgentConfig.LLMCallTimeout or the default.
|
||||
func (e *AgentEngine) getLLMStallTimeout() time.Duration {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLLMRetryDelayHonoursRetryAfter(t *testing.T) {
|
||||
limited := func(retryAfter string) error {
|
||||
return fmt.Errorf("chat: %w", &api.HTTPError{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Header: http.Header{"Retry-After": []string{retryAfter}},
|
||||
})
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
attempt int
|
||||
want time.Duration
|
||||
}{
|
||||
{"no header keeps the linear backoff", &api.HTTPError{StatusCode: 503}, 2, 2 * time.Second},
|
||||
{"untyped error keeps the linear backoff", errors.New("timeout"), 1, time.Second},
|
||||
{"retry-after longer than the backoff wins", limited("7"), 1, 7 * time.Second},
|
||||
{"retry-after shorter than the backoff is not a speed-up", limited("1"), 3, 3 * time.Second},
|
||||
{"retry-after is capped", limited("3600"), 1, maxLLMRetryAfter},
|
||||
{"malformed retry-after is ignored", limited("soon"), 1, time.Second},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, llmRetryDelay(tc.err, tc.attempt))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -571,7 +571,7 @@ func (e *AgentEngine) callLLMWithRetry(
|
|||
if err != nil && isTransientError(err) {
|
||||
// Retry transient errors (timeout, rate limit, server errors) up to maxLLMRetries times
|
||||
for retry := 1; retry <= maxLLMRetries; retry++ {
|
||||
retryDelay := time.Duration(retry) * time.Second
|
||||
retryDelay := llmRetryDelay(err, retry)
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] LLM transient error (attempt %d/%d), retrying in %v: %v",
|
||||
round, retry, maxLLMRetries, retryDelay, err)
|
||||
// A stop pressed during the backoff ends the turn now rather than
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
|
|
@ -139,7 +141,7 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
|
|||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
_ = resp.Body.Close()
|
||||
return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body)}
|
||||
return nil, &HTTPError{StatusCode: resp.StatusCode, Body: string(body), Header: resp.Header}
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
|
@ -201,12 +203,45 @@ func (e *TransportError) Unwrap() error { return e.Err }
|
|||
type HTTPError struct {
|
||||
StatusCode int
|
||||
Body string
|
||||
// Header is the reply's headers, kept so a retry can honour the vendor's
|
||||
// Retry-After on a 429 or 503 instead of guessing its own backoff.
|
||||
Header http.Header
|
||||
}
|
||||
|
||||
func (e *HTTPError) Error() string {
|
||||
return fmt.Sprintf("API request failed with status %d: %s", e.StatusCode, e.Body)
|
||||
}
|
||||
|
||||
// RetryAfter is how long the vendor asked the caller to wait before trying
|
||||
// again, from the Retry-After header in either of its two forms (delay in
|
||||
// seconds or an HTTP date). It is zero when the header is absent, malformed
|
||||
// or already in the past.
|
||||
func (e *HTTPError) RetryAfter() time.Duration {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return parseRetryAfter(e.Header.Get("Retry-After"), time.Now())
|
||||
}
|
||||
|
||||
func parseRetryAfter(value string, now time.Time) time.Duration {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return 0
|
||||
}
|
||||
if secs, err := strconv.Atoi(value); err == nil {
|
||||
if secs <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
if at, err := http.ParseTime(value); err == nil {
|
||||
if d := at.Sub(now); d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// LogRequest emits the standard request log line with image payloads
|
||||
// compacted so data URIs do not flood the log.
|
||||
func LogRequest(ctx context.Context, url, model string, body []byte, stream bool) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDoKeepsHeadersOnHTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "12")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = w.Write([]byte(`{"error":"rate limited"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, srv.URL, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = Endpoint{Client: srv.Client()}.Do(req)
|
||||
|
||||
var httpErr *HTTPError
|
||||
require.True(t, errors.As(err, &httpErr))
|
||||
require.Equal(t, http.StatusTooManyRequests, httpErr.StatusCode)
|
||||
require.Equal(t, 12*time.Second, httpErr.RetryAfter())
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
now := time.Date(2026, 9, 26, 12, 0, 0, 0, time.UTC)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
value string
|
||||
want time.Duration
|
||||
}{
|
||||
{"absent", "", 0},
|
||||
{"seconds", "30", 30 * time.Second},
|
||||
{"seconds with spaces", " 5 ", 5 * time.Second},
|
||||
{"zero", "0", 0},
|
||||
{"negative", "-3", 0},
|
||||
{"http date", now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second},
|
||||
{"http date in the past", now.Add(-time.Minute).Format(http.TimeFormat), 0},
|
||||
{"garbage", "tomorrow", 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, parseRetryAfter(tc.value, now))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterOnNilError(t *testing.T) {
|
||||
var e *HTTPError
|
||||
require.Zero(t, e.RetryAfter())
|
||||
require.Zero(t, (&HTTPError{StatusCode: 503}).RetryAfter())
|
||||
}
|
||||
Loading…
Reference in New Issue