From d823ef61ae3af1671d657b3706ba1c693f139e19 Mon Sep 17 00:00:00 2001 From: Rene Zander Date: Sat, 8 Aug 2026 12:13:33 +0000 Subject: [PATCH] fix(vlm): use max_completion_tokens for GPT-5 / o-series vision models The VLM path sent max_tokens and temperature, both rejected by reasoning and GPT-5 models. go-openai's reasoning validator failed the request client-side, so every OCR and caption call errored with "this model is not supported MaxTokens, please use MaxCompletionTokens" and no image chunk was ever created. The chat path already solved this in shapeOpenAIReasoning (issue #1283); the VLM path was never wired to it. Reuse the shared provider.IsOpenAIReasoningOrGPT5Model predicate so both paths agree on which models are affected. Migrating max_tokens alone is not enough -- the VLM default temperature of 0.1 is independently rejected. Also report an empty completion with finish_reason=length as an error. max_completion_tokens covers reasoning tokens, so an exhausted budget returns empty content rather than an API error, which was previously recorded as no_extracted_content and looked identical to an image with no text. Fixes #2537 Co-Authored-By: Claude Opus 5 (1M context) --- internal/models/vlm/remote_api.go | 39 ++- .../models/vlm/remote_api_reasoning_test.go | 293 ++++++++++++++++++ 2 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 internal/models/vlm/remote_api_reasoning_test.go diff --git a/internal/models/vlm/remote_api.go b/internal/models/vlm/remote_api.go index 6b511282..2544d11a 100644 --- a/internal/models/vlm/remote_api.go +++ b/internal/models/vlm/remote_api.go @@ -143,6 +143,7 @@ func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, promp MaxTokens: defaultMaxToks, Temperature: v.temperature, } + shapeReasoningVLMRequest(&req) totalImageSize := 0 for _, img := range imgBytesList { @@ -159,11 +160,47 @@ func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, promp return "", fmt.Errorf("OpenAI VLM returned no choices") } - content := resp.Choices[0].Message.Content + choice := resp.Choices[0] + content := choice.Message.Content + if strings.TrimSpace(content) == "" && choice.FinishReason == openai.FinishReasonLength { + // Reasoning models spend max_completion_tokens on reasoning before any + // visible output, so an exhausted budget yields an empty message rather + // than an API error. Returning "" here would be recorded as + // "no_extracted_content" and look identical to an image with no text. + return "", fmt.Errorf( + "OpenAI VLM returned no content: completion truncated at %d tokens (finish_reason=length)", + defaultMaxToks, + ) + } logger.Infof(ctx, "[VLM] OpenAI response received, len=%d", len(content)) return content, nil } +// shapeReasoningVLMRequest adapts an OpenAI-compatible VLM request for +// reasoning (o-series) and GPT-5 models, which reject `max_tokens` and every +// non-default sampling parameter. +// +// This mirrors shapeOpenAIReasoning in internal/models/chat, which fixed the +// same incompatibility on the chat path for issue #1283. The VLM path was +// never wired to it, so image OCR and captioning failed for every one of these +// models (issue #2537). +// +// Both quirks have to be handled together: migrating max_tokens alone still +// fails, because the VLM default temperature (0.1) is itself rejected. +func shapeReasoningVLMRequest(req *openai.ChatCompletionRequest) { + if !provider.IsOpenAIReasoningOrGPT5Model(req.Model) { + return + } + if req.MaxCompletionTokens == 0 && req.MaxTokens > 0 { + req.MaxCompletionTokens = req.MaxTokens + } + req.MaxTokens = 0 + req.Temperature = 0 + req.TopP = 0 + req.FrequencyPenalty = 0 + req.PresencePenalty = 0 +} + func (v *RemoteAPIVLM) GetModelName() string { return v.modelName } func (v *RemoteAPIVLM) GetModelID() string { return v.modelID } diff --git a/internal/models/vlm/remote_api_reasoning_test.go b/internal/models/vlm/remote_api_reasoning_test.go new file mode 100644 index 00000000..c376abbd --- /dev/null +++ b/internal/models/vlm/remote_api_reasoning_test.go @@ -0,0 +1,293 @@ +package vlm + +import ( + "encoding/json" + "errors" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + + openai "github.com/sashabaranov/go-openai" +) + +// TestShapeReasoningVLMRequest 验证 GPT-5 / o-series 的请求参数改写。 +// 见 issue #2537:这些模型必须使用 max_completion_tokens 替代 max_tokens, +// 且不接受非默认的采样参数。 +func TestShapeReasoningVLMRequest(t *testing.T) { + cases := []struct { + name string + model string + maxTokens int + maxCompletionTokens int + temperature float32 + wantMaxTokens int + wantMaxCompletionTokens int + wantTemperature float32 + }{ + { + name: "gpt-5 migrates max_tokens and drops temperature", + model: "gpt-5", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "gpt-5-nano is shaped", // the model reported in issue #2537 + model: "gpt-5-nano", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "gpt-5 mixed case is shaped", + model: "GPT-5.4-Mini", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "o1-mini is shaped", + model: "o1-mini", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "o3 is shaped", + model: "o3", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "o4-mini is shaped", + model: "o4-mini", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0, + }, + { + name: "explicit max_completion_tokens is preserved", + model: "gpt-5", maxTokens: 5000, maxCompletionTokens: 128, temperature: 0.1, + wantMaxTokens: 0, wantMaxCompletionTokens: 128, wantTemperature: 0, + }, + { + name: "gpt-4o is left untouched", + model: "gpt-4o", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1, + }, + { + name: "qwen-vl is left untouched", + model: "qwen2.5-vl-7b-instruct", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1, + }, + { + name: "empty model is left untouched", + model: "", maxTokens: 5000, temperature: 0.1, + wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := openai.ChatCompletionRequest{ + Model: tc.model, + MaxTokens: tc.maxTokens, + MaxCompletionTokens: tc.maxCompletionTokens, + Temperature: tc.temperature, + } + shapeReasoningVLMRequest(&req) + + if req.MaxTokens != tc.wantMaxTokens { + t.Errorf("MaxTokens = %d, want %d", req.MaxTokens, tc.wantMaxTokens) + } + if req.MaxCompletionTokens != tc.wantMaxCompletionTokens { + t.Errorf("MaxCompletionTokens = %d, want %d", req.MaxCompletionTokens, tc.wantMaxCompletionTokens) + } + if req.Temperature != tc.wantTemperature { + t.Errorf("Temperature = %v, want %v", req.Temperature, tc.wantTemperature) + } + }) + } +} + +// newVLMChatTestServer emulates an OpenAI-compatible chat completions endpoint +// and records the last decoded request body. +func newVLMChatTestServer(t *testing.T, lastRequest *map[string]interface{}) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode VLM request: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + *lastRequest = req + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl-test", + "object": "chat.completion", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "extracted text"}, "finish_reason": "stop"} + ] + }`)) + })) +} + +// testPNG is a minimal byte slice that http.DetectContentType reports as a PNG. +var testPNG = []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 16)) + +// TestRemoteAPIVLMSendsMaxCompletionTokensForReasoningModel is the regression +// test for issue #2537: with a GPT-5 / o-series vision model, every OCR and +// caption call failed with +// +// "this model is not supported MaxTokens, please use MaxCompletionTokens" +// +// The request was rejected client-side by go-openai's reasoning validator, so +// it never reached the server and no image chunk was ever created. +func TestRemoteAPIVLMSendsMaxCompletionTokensForReasoningModel(t *testing.T) { + withVLMSSRFWhitelist(t, "127.0.0.1") + + var lastRequest map[string]interface{} + server := newVLMChatTestServer(t, &lastRequest) + defer server.Close() + + v, err := NewRemoteAPIVLM(&Config{ + BaseURL: server.URL, + ModelName: "gpt-5-nano", + APIKey: "sk-test", + }) + if err != nil { + t.Fatalf("NewRemoteAPIVLM: %v", err) + } + + content, err := v.Predict(t.Context(), [][]byte{testPNG}, "extract the text") + if err != nil { + t.Fatalf("Predict: %v", err) + } + if content != "extracted text" { + t.Errorf("content = %q, want %q", content, "extracted text") + } + + if _, ok := lastRequest["max_tokens"]; ok { + t.Errorf("request carries max_tokens, which reasoning models reject: %v", lastRequest["max_tokens"]) + } + if got, ok := lastRequest["max_completion_tokens"]; !ok { + t.Error("request is missing max_completion_tokens") + } else if got != float64(defaultMaxToks) { + t.Errorf("max_completion_tokens = %v, want %d", got, defaultMaxToks) + } + // Temperature 0.1 is itself rejected for these models, so migrating + // max_tokens alone would not have been enough. + if _, ok := lastRequest["temperature"]; ok { + t.Errorf("request carries temperature, which reasoning models reject: %v", lastRequest["temperature"]) + } +} + +// TestRemoteAPIVLMKeepsMaxTokensForNonReasoningModel guards against the fix +// regressing ordinary vision models, which still expect max_tokens. +func TestRemoteAPIVLMKeepsMaxTokensForNonReasoningModel(t *testing.T) { + withVLMSSRFWhitelist(t, "127.0.0.1") + + var lastRequest map[string]interface{} + server := newVLMChatTestServer(t, &lastRequest) + defer server.Close() + + v, err := NewRemoteAPIVLM(&Config{ + BaseURL: server.URL, + ModelName: "gpt-4o", + APIKey: "sk-test", + }) + if err != nil { + t.Fatalf("NewRemoteAPIVLM: %v", err) + } + + if _, err := v.Predict(t.Context(), [][]byte{testPNG}, "extract the text"); err != nil { + t.Fatalf("Predict: %v", err) + } + + if got, ok := lastRequest["max_tokens"]; !ok { + t.Error("request is missing max_tokens") + } else if got != float64(defaultMaxToks) { + t.Errorf("max_tokens = %v, want %d", got, defaultMaxToks) + } + if _, ok := lastRequest["max_completion_tokens"]; ok { + t.Error("request carries max_completion_tokens for a non-reasoning model") + } + if got, ok := lastRequest["temperature"]; !ok { + t.Error("request is missing temperature") + } else if f, isFloat := got.(float64); !isFloat || math.Abs(f-float64(defaultTemp)) > 1e-6 { + t.Errorf("temperature = %v, want %v", got, defaultTemp) + } +} + +// TestRemoteAPIVLMReportsTruncatedCompletion covers the other way a reasoning +// model yields nothing: max_completion_tokens also covers reasoning tokens, so +// an exhausted budget returns an empty message with finish_reason=length +// instead of an API error. Reporting that as an error keeps it out of the +// "no_extracted_content" bucket, where issue #2537 notes the failure is +// indistinguishable from an image that genuinely has no text. +func TestRemoteAPIVLMReportsTruncatedCompletion(t *testing.T) { + withVLMSSRFWhitelist(t, "127.0.0.1") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl-test", + "object": "chat.completion", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"} + ] + }`)) + })) + defer server.Close() + + v, err := NewRemoteAPIVLM(&Config{ + BaseURL: server.URL, + ModelName: "gpt-5-nano", + APIKey: "sk-test", + }) + if err != nil { + t.Fatalf("NewRemoteAPIVLM: %v", err) + } + + _, err = v.Predict(t.Context(), [][]byte{testPNG}, "extract the text") + if err == nil { + t.Fatal("Predict returned nil error for a truncated completion") + } + if !strings.Contains(err.Error(), "truncated") { + t.Errorf("error = %q, want it to mention truncation", err.Error()) + } +} + +// TestRemoteAPIVLMUnshapedReasoningRequestIsRejected pins the upstream +// behavior this fix works around: without the shaping, go-openai rejects the +// request before it leaves the process. It fails identically for max_tokens +// and for a non-default temperature. +func TestRemoteAPIVLMUnshapedReasoningRequestIsRejected(t *testing.T) { + withVLMSSRFWhitelist(t, "127.0.0.1") + + var lastRequest map[string]interface{} + server := newVLMChatTestServer(t, &lastRequest) + defer server.Close() + + v, err := NewRemoteAPIVLM(&Config{ + BaseURL: server.URL, + ModelName: "gpt-5-nano", + APIKey: "sk-test", + }) + if err != nil { + t.Fatalf("NewRemoteAPIVLM: %v", err) + } + + unshaped := openai.ChatCompletionRequest{ + Model: "gpt-5-nano", + Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleUser, Content: "hi"}}, + MaxTokens: defaultMaxToks, + } + _, err = v.client.CreateChatCompletion(t.Context(), unshaped) + if !errors.Is(err, openai.ErrReasoningModelMaxTokensDeprecated) { + t.Errorf("max_tokens error = %v, want ErrReasoningModelMaxTokensDeprecated", err) + } + + tempOnly := openai.ChatCompletionRequest{ + Model: "gpt-5-nano", + Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleUser, Content: "hi"}}, + Temperature: defaultTemp, + } + _, err = v.client.CreateChatCompletion(t.Context(), tempOnly) + if !errors.Is(err, openai.ErrReasoningModelLimitationsOther) { + t.Errorf("temperature error = %v, want ErrReasoningModelLimitationsOther", err) + } +}