WeKnora/internal/models/vlm/remote_api.go

143 lines
3.8 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package vlm
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/provider"
secutils "github.com/Tencent/WeKnora/internal/utils"
openai "github.com/sashabaranov/go-openai"
)
const (
defaultTimeout = 90 * time.Second
defaultMaxToks = 5000
defaultTemp = float32(0.1)
)
// RemoteAPIVLM implements VLM via an OpenAI-compatible chat completions API.
type RemoteAPIVLM struct {
modelName string
modelID string
client *openai.Client
baseURL string
}
// NewRemoteAPIVLM creates a remote-API backed VLM instance.
func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
providerName := provider.ProviderName(config.Provider)
if providerName == "" {
providerName = provider.DetectProvider(config.BaseURL)
}
var apiCfg openai.ClientConfig
if providerName == provider.ProviderAzureOpenAI {
apiCfg = openai.DefaultAzureConfig(config.APIKey, config.BaseURL)
apiCfg.AzureModelMapperFunc = func(model string) string {
return model
}
if config.Extra != nil {
if v, ok := config.Extra["api_version"]; ok {
if vs, ok := v.(string); ok && vs != "" {
apiCfg.APIVersion = vs
}
}
}
} else {
apiCfg = openai.DefaultConfig(config.APIKey)
if config.BaseURL != "" {
apiCfg.BaseURL = config.BaseURL
}
}
httpClient := &http.Client{Timeout: defaultTimeout}
// 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers)
if len(config.CustomHeaders) > 0 {
apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)
} else {
apiCfg.HTTPClient = httpClient
}
return &RemoteAPIVLM{
modelName: config.ModelName,
modelID: config.ModelID,
client: openai.NewClientWithConfig(apiCfg),
baseURL: config.BaseURL,
}, nil
}
// Predict sends an image with a text prompt to the OpenAI-compatible API.
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
var parts []openai.ChatMessagePart
// Add text prompt first
parts = append(parts, openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeText,
Text: prompt,
})
// Add images
for _, imgBytes := range imgBytesList {
if len(imgBytes) > 0 {
mimeType := detectImageMIME(imgBytes)
b64 := base64.StdEncoding.EncodeToString(imgBytes)
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
parts = append(parts, openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: dataURI,
Detail: openai.ImageURLDetailAuto,
},
})
}
}
req := openai.ChatCompletionRequest{
Model: v.modelName,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleUser,
MultiContent: parts,
},
},
MaxTokens: defaultMaxToks,
Temperature: defaultTemp,
}
totalImageSize := 0
for _, img := range imgBytesList {
totalImageSize += len(img)
}
logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, numImages=%d, totalImageSize=%d",
v.modelName, v.baseURL, len(imgBytesList), totalImageSize)
resp, err := v.client.CreateChatCompletion(ctx, req)
if err != nil {
return "", fmt.Errorf("OpenAI VLM request: %w", err)
}
if len(resp.Choices) == 0 {
return "", fmt.Errorf("OpenAI VLM returned no choices")
}
content := resp.Choices[0].Message.Content
logger.Infof(ctx, "[VLM] OpenAI response received, len=%d", len(content))
return content, nil
}
func (v *RemoteAPIVLM) GetModelName() string { return v.modelName }
func (v *RemoteAPIVLM) GetModelID() string { return v.modelID }
// detectImageMIME returns the MIME type for the given image bytes.
func detectImageMIME(data []byte) string {
ct := http.DetectContentType(data)
if strings.HasPrefix(ct, "image/") {
return ct
}
return "image/png"
}