fix(embed): 内嵌网页只传图片不输入文字时不再返回 400

内嵌网页的输入框允许只带图片或附件就点击发送,但 CreateKnowledgeQARequest.Query
带有 binding:"required",parseQARequest 也拒绝空 query,于是只传图片直接返回
400 "Query content cannot be empty"。

入口处理:去掉 binding:"required";文字为空但带有内联图片数据或内联附件时,
用 types.UploadOnlyQuestion 生成一句替用户提问的问题(中文界面为「请根据我
上传的内容回答。」,其他语言为英文),交给模型、检索、标题、会话历史索引、
追问建议和记忆使用。只有 URL 的图片不算上传,因为客户端传入的图片 URL 会被
清掉;预上传的 attachment_ids 也不算,这类文件在流开始后才解析,可能失败或
超时,届时模型没有任何内容可答。其余空 query 仍返回 400。

存储与显示:qaRequestContext 新增 userInput,保存用户消息时只存用户实际
输入,只传图片时为空,刷新后与发送当下显示一致;query 仍是给模型的问题。
steer 追问复制上一轮的请求上下文,显式设置 userInput,避免在只传图片的一轮
之后把追问存成空消息。

会话历史:文字为空但带图片或附件的用户消息,在两处历史重建里补上同一句
问题。知识问答流水线(loadAndProcessHistory)原先会整轮丢弃;Agent 历史
(LoadAgentHistory)原先会发出空的用户消息,被 SanitizeMessages 剔除后
前后两条回答被合并。

去掉 binding 标签会让 gofmt 重新对齐整个 CreateKnowledgeQARequest 的行尾
注释,这些既有的超长行因此会被 PR 的增量 lint 视为新增。按仓库惯例把字段
注释移到字段上一行(注释文字不变,swagger 描述不受影响),并把 Go 字段
KnowledgeIds 改名为 KnowledgeIDs(JSON 名仍是 knowledge_ids,接口不变)。

同步更新 swagger 文档,query 不再是必填字段。
main
hailongzhao 2026-09-29 19:08:44 +08:00 committed by lyingbug
parent 75fc0ad159
commit bccb4b151b
13 changed files with 243 additions and 36 deletions

View File

@ -26121,9 +26121,6 @@ const docTemplate = `{
},
"internal_handler_session.CreateKnowledgeQARequest": {
"type": "object",
"required": [
"query"
],
"properties": {
"agent_enabled": {
"description": "Whether agent mode is enabled for this request",
@ -26199,7 +26196,7 @@ const docTemplate = `{
}
},
"query": {
"description": "Query text for knowledge base search",
"description": "Query text; may be empty only when an image or file is attached",
"type": "string"
},
"question_origin": {

View File

@ -26114,9 +26114,6 @@
},
"internal_handler_session.CreateKnowledgeQARequest": {
"type": "object",
"required": [
"query"
],
"properties": {
"agent_enabled": {
"description": "Whether agent mode is enabled for this request",
@ -26192,7 +26189,7 @@
}
},
"query": {
"description": "Query text for knowledge base search",
"description": "Query text; may be empty only when an image or file is attached",
"type": "string"
},
"question_origin": {

View File

@ -6435,7 +6435,7 @@ definitions:
$ref: '#/definitions/internal_handler_session.MentionedItemRequest'
type: array
query:
description: Query text for knowledge base search
description: Query text; may be empty only when an image or file is attached
type: string
question_origin:
allOf:
@ -6461,8 +6461,6 @@ definitions:
web_search_enabled:
description: Whether web search is enabled for this request
type: boolean
required:
- query
type: object
internal_handler_session.CreateSessionRequest:
properties:

View File

@ -104,6 +104,7 @@ func LoadAgentHistory(
used int
replay = newHistoryReplay(estimator, 0, retainRetrievalHistory)
)
replay.locale = types.LanguageFromContextOrDefault(ctx)
for len(rows) < agentHistoryMaxRows {
page, err := messageRepo.ListMessagesBySessionBeforeCursor(
ctx, sessionID, before, beforeID, agentHistoryPageSize,
@ -259,6 +260,7 @@ type historyReplay struct {
estimator *agenttoken.Estimator
budget int
retainRetrievalHistory bool
locale string
built map[string][]chat.Message
tokens map[string]int
// full holds stored rows by ID until their turn is replayed.
@ -305,7 +307,7 @@ func (r *historyReplay) messages(t *agentHistoryTurn) []chat.Message {
for i, u := range t.users {
users[i] = r.stored(u)
}
msgs := append([]chat.Message{buildUserHistoryMessage(users[0])},
msgs := append([]chat.Message{r.openingUserMessage(users[0])},
buildTurnBodyMessages(r.stored(t.assistant), users[1:])...)
for i := range msgs {
msgs[i].TurnID = id
@ -464,6 +466,18 @@ func buildTurnBodyMessages(assistant *types.Message, midRunUsers []*types.Messag
return out
}
// openingUserMessage replays the message that started a turn. One sent as only
// an image or file has no stored text; replayed empty, the sanitizer would drop
// it and merge the answers on either side.
func (r *historyReplay) openingUserMessage(m *types.Message) chat.Message {
if strings.TrimSpace(m.Content) == "" && (len(m.Images) > 0 || len(m.Attachments) > 0) {
asked := *m
asked.Content = types.UploadOnlyQuestion(r.locale)
return buildUserHistoryMessage(&asked)
}
return buildUserHistoryMessage(m)
}
// buildUserHistoryMessage converts a stored user message into the chat.Message
// form that should appear in LLM history. It deliberately ignores
// RenderedContent: that field is a snapshot of the old prompt and retrieval

View File

@ -132,6 +132,21 @@ func TestLoadAgentHistoryTagsEveryMessageWithItsTurn(t *testing.T) {
}
}
// A turn sent as only an image has no stored text. Replaying it empty lets the
// sanitizer drop the user message and merge the two answers around it.
func TestLoadAgentHistoryAsksTheUploadOnlyQuestionForAnImageOnlyTurn(t *testing.T) {
rows := storedTurns(2)
rows[0].Content = ""
rows[0].Images = types.MessageImages{{URL: "resource://img"}}
repo := &historyRepo{rows: rows}
ctx := context.WithValue(context.Background(), types.LanguageContextKey, "en-US")
got, _, err := LoadAgentHistory(ctx, repo, "s1", unlimitedBudget, false)
require.NoError(t, err)
require.Equal(t, []string{types.UploadOnlyQuestion("en-US"), "question 2"}, questions(got))
}
// The turn count no longer bounds history: every turn that fits the budget is
// replayed, however many there are.
func TestLoadAgentHistoryIsNotCappedByTurnCount(t *testing.T) {

View File

@ -152,6 +152,9 @@ func loadAndProcessHistory(
// Historical references are carried separately in KnowledgeReferences
// and can be re-merged into this turn's freshly rendered context.
h.Query = message.Content
if strings.TrimSpace(h.Query) == "" && (len(message.Images) > 0 || len(message.Attachments) > 0) {
h.Query = types.UploadOnlyQuestion(types.LanguageFromContextOrDefault(ctx))
}
h.CreateAt = message.CreatedAt
if desc := extractImageCaptions(message.Images); desc != "" {
h.Query += "\n\n[用户上传图片内容]\n" + desc

View File

@ -0,0 +1,37 @@
package chatpipeline
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type recentMessagesStub struct {
interfaces.MessageService
messages []*types.Message
}
func (s *recentMessagesStub) GetRecentMessagesBySession(context.Context, string, int) ([]*types.Message, error) {
return s.messages, nil
}
func TestLoadAndProcessHistoryKeepsImageOnlyTurn(t *testing.T) {
at := time.Date(2026, 9, 29, 10, 0, 0, 0, time.UTC)
svc := &recentMessagesStub{messages: []*types.Message{
{RequestID: "r1", Role: "user", Images: types.MessageImages{{URL: "resource://img"}}, CreatedAt: at},
{RequestID: "r1", Role: "assistant", Content: "a cat", CreatedAt: at.Add(time.Second)},
}}
ctx := context.WithValue(context.Background(), types.LanguageContextKey, "zh-CN")
history, err := loadAndProcessHistory(ctx, svc, "s1", 5, 20)
require.NoError(t, err)
require.Len(t, history, 1, "an image-only turn must stay in history for follow-up questions")
assert.Equal(t, types.UploadOnlyQuestion("zh-CN"), history[0].Query)
assert.Equal(t, "a cat", history[0].Answer)
}

View File

@ -43,7 +43,8 @@ type qaRequestContext struct {
sessionID string
requestID string
receivedAt time.Time // Wall-clock time the handler started processing the request
query string
query string // Question sent to models; the upload-only question when the user typed nothing
userInput string // Text the user typed, stored on the user message; may be empty
session *types.Session
customAgent *types.CustomAgent
assistantMessage *types.Message
@ -132,6 +133,25 @@ func (rc *qaRequestContext) buildQARequest() *types.QARequest {
return req
}
// uploadOnlyQuery returns the question used for a request that carries inline
// image or file content but no text, or "" when it carries neither. URL-only
// images do not count: parseQARequest strips client-supplied image URLs.
// Pre-uploaded attachment IDs do not count either: those documents can still
// fail or time out after the stream starts, leaving nothing to answer from.
func uploadOnlyQuery(ctx context.Context, req *CreateKnowledgeQARequest) string {
hasUpload := false
for _, img := range req.Images {
hasUpload = hasUpload || strings.TrimSpace(img.Data) != ""
}
for _, att := range req.AttachmentUploads {
hasUpload = hasUpload || strings.TrimSpace(att.Data) != ""
}
if !hasUpload {
return ""
}
return types.UploadOnlyQuestion(types.LanguageFromContextOrDefault(ctx))
}
// parseQARequest parses and validates a QA request, returns the request context
func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestContext, *CreateKnowledgeQARequest, error) {
receivedAt := time.Now()
@ -170,9 +190,14 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
logger.Error(ctx, "Query content is invalid")
return nil, nil, errors.NewBadRequestError("Query content contains invalid content")
}
modelQuery := request.Query
if validatedQuery == "" {
logger.Error(ctx, "Query content is empty")
return nil, nil, errors.NewBadRequestError("Query content cannot be empty")
modelQuery = uploadOnlyQuery(ctx, &request)
if modelQuery == "" {
logger.Error(ctx, "Query content is empty")
return nil, nil, errors.NewBadRequestError("Query content cannot be empty")
}
request.Query = ""
}
// Resolve the storage-reference representation up front: once the SSE stream
@ -219,7 +244,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
}
// Merge @mentioned items into knowledge_base_ids and knowledge_ids
kbIDs, knowledgeIDs := mergeKnowledgeTargets(request.KnowledgeBaseIDs, request.KnowledgeIds, request.MentionedItems)
kbIDs, knowledgeIDs := mergeKnowledgeTargets(request.KnowledgeBaseIDs, request.KnowledgeIDs, request.MentionedItems)
if err := types.AuthorizeTenantAPIKeyKnowledgeTargets(ctx, kbIDs, knowledgeIDs); err != nil {
return nil, nil, err
}
@ -421,7 +446,8 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
sessionID: sessionID,
requestID: requestID,
receivedAt: receivedAt,
query: request.Query,
query: modelQuery,
userInput: request.Query,
session: session,
customAgent: customAgent,
assistantMessage: &types.Message{
@ -1064,7 +1090,7 @@ func (h *Handler) persistTurnMessages(ctx context.Context, reqCtx *qaRequestCont
userMsg, err := h.createUserMessage(
ctx,
reqCtx.sessionID,
reqCtx.query,
reqCtx.userInput,
reqCtx.requestID,
reqCtx.mentionedItems,
convertImageAttachments(reqCtx.images),

View File

@ -82,3 +82,92 @@ func TestQAQueryValidation(t *testing.T) {
})
}
}
func TestQAQueryValidationAcceptsUploadWithoutText(t *testing.T) {
gin.SetMode(gin.TestMode)
image := []ImageAttachment{{Data: "data:image/png;base64,aGk="}}
tests := []struct {
name string
body CreateKnowledgeQARequest
rejectBeforeLookup bool
}{
{name: "image only", body: CreateKnowledgeQARequest{Images: image}},
{name: "whitespace with image", body: CreateKnowledgeQARequest{Query: " \n", Images: image}},
{
name: "attachment upload only",
body: CreateKnowledgeQARequest{AttachmentUploads: []AttachmentUpload{{Data: "aGk=", FileName: "a.txt"}}},
},
{
name: "url-only image",
body: CreateKnowledgeQARequest{Images: []ImageAttachment{{URL: "http://127.0.0.1/a.png"}}},
rejectBeforeLookup: true,
},
// Pre-uploaded documents may still fail or time out after the stream
// starts, leaving the turn with no content to answer from.
{
name: "attachment id only",
body: CreateKnowledgeQARequest{AttachmentIDs: []string{"doc-1"}},
rejectBeforeLookup: true,
},
}
for _, endpoint := range []string{"knowledge-chat", "agent-chat"} {
t.Run(endpoint, func(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sessions := &queryValidationSessionStub{}
h := &Handler{sessionService: sessions}
router := gin.New()
router.Use(middleware.ErrorHandler())
router.POST("/knowledge-chat/:session_id", h.KnowledgeQA)
router.POST("/agent-chat/:session_id", h.AgentQA)
body, err := json.Marshal(tt.body)
require.NoError(t, err)
request := httptest.NewRequest(http.MethodPost, "/"+endpoint+"/session-1", bytes.NewReader(body))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
router.ServeHTTP(response, request)
if tt.rejectBeforeLookup {
assert.Equal(t, http.StatusBadRequest, response.Code, response.Body.String())
assert.Zero(t, sessions.lookupCalls)
} else {
assert.Equal(t, http.StatusNotFound, response.Code, response.Body.String())
assert.Equal(t, 1, sessions.lookupCalls)
}
})
}
})
}
}
func TestPersistTurnMessagesStoresWhatTheUserTyped(t *testing.T) {
msgs := &steerPersistingMessageStub{}
h := &Handler{messageService: msgs}
question := types.UploadOnlyQuestion("zh-CN")
reqCtx := &qaRequestContext{
sessionID: "sess-1",
query: question,
images: []ImageAttachment{{Data: "data:image/png;base64,aGk=", URL: "resource://img"}},
assistantMessage: &types.Message{SessionID: "sess-1", Role: "assistant"},
}
require.NoError(t, h.persistTurnMessages(context.Background(), reqCtx))
stored := msgs.byID[reqCtx.userMessageID]
require.NotNil(t, stored)
assert.Empty(t, stored.Content, "an image-only message must not gain text the user never typed")
assert.Len(t, stored.Images, 1)
assert.Equal(t, question, reqCtx.buildQARequest().Query)
}
func TestUploadOnlyQuery(t *testing.T) {
image := &CreateKnowledgeQARequest{Images: []ImageAttachment{{Data: "data:image/png;base64,aGk="}}}
zh := context.WithValue(context.Background(), types.LanguageContextKey, "zh-CN")
en := context.WithValue(context.Background(), types.LanguageContextKey, "en-US")
assert.Equal(t, "请根据我上传的内容回答。", uploadOnlyQuery(zh, image))
assert.Equal(t, "Please answer based on what I uploaded.", uploadOnlyQuery(en, image))
assert.Empty(t, uploadOnlyQuery(zh, &CreateKnowledgeQARequest{}))
assert.Empty(t, uploadOnlyQuery(zh, &CreateKnowledgeQARequest{Images: []ImageAttachment{{URL: "http://x/a.png"}}}))
}

View File

@ -918,6 +918,7 @@ func (h *Handler) claimNextSteerFollowUp(
followUp := *prevReqCtx
followUp.ctx = ctx
followUp.query = first.Content
followUp.userInput = first.Content
followUp.requestID = uuid.New().String()
followUp.channel = getString(first.Data, "channel")
if followUp.channel == "" {

View File

@ -249,6 +249,7 @@ func TestSteerFollowUpHandoffKeepsSessionLiveAcrossPreviousClear(t *testing.T) {
require.True(t, ok)
require.NotNil(t, followUp)
assert.Equal(t, "do this next", followUp.query)
assert.Equal(t, "do this next", msgs.byID[followUp.userMessageID].Content)
require.NotEmpty(t, followUp.assistantMessage.ID)
assert.NotEqual(t, "assist-A", followUp.assistantMessage.ID)
assert.Empty(t, followUp.steerCarryOver, "carry-over must already sit on the new run's list")

View File

@ -46,24 +46,42 @@ type ImageAttachment struct {
// CreateKnowledgeQARequest defines the request structure for knowledge QA
type CreateKnowledgeQARequest struct {
Query string `json:"query" binding:"required"` // Query text for knowledge base search
KnowledgeBaseIDs []string `json:"knowledge_base_ids"` // Selected knowledge base ID for this request
KnowledgeIds []string `json:"knowledge_ids"` // Selected knowledge ID for this request
AgentEnabled bool `json:"agent_enabled"` // Whether agent mode is enabled for this request
AgentID string `json:"agent_id"` // Selected custom agent ID (backend resolves shared agent and its workspace from share relation)
AgentSourceTenantID uint64 `json:"agent_source_tenant_id,omitempty"` // Optional disambiguator; backend still verifies the share relation
LocalBrowserEnabled bool `json:"local_browser_enabled"` // Browser source
WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this request
SummaryModelID string `json:"summary_model_id"` // Optional summary model ID for this request (overrides session default)
MCPServiceIDs []string `json:"mcp_service_ids"` // Per-request MCP services selected via @mention
SkillNames []string `json:"skill_names"` // Per-request Skills selected via @mention
TagIDs []string `json:"tag_ids"` // @mentioned tag IDs (display/debug; scoped via MentionedItems)
MentionedItems []MentionedItemRequest `json:"mentioned_items"` // @mentioned knowledge bases and files
DisableTitle bool `json:"disable_title"` // Whether to disable auto title generation
Images []ImageAttachment `json:"images"` // Attached images for multimodal chat
AttachmentUploads []AttachmentUpload `json:"attachment_uploads,omitempty"` // Attached files (documents, audio, etc.)
AttachmentIDs []string `json:"attachment_ids,omitempty"` // Pre-uploaded session-scoped document IDs
Channel string `json:"channel"` // Source channel: "web", "api", "im", etc.
// Query text; may be empty only when an image or file is attached
Query string `json:"query"`
// Selected knowledge base ID for this request
KnowledgeBaseIDs []string `json:"knowledge_base_ids"`
// Selected knowledge ID for this request
KnowledgeIDs []string `json:"knowledge_ids"`
// Whether agent mode is enabled for this request
AgentEnabled bool `json:"agent_enabled"`
// Selected custom agent ID (backend resolves shared agent and its workspace from share relation)
AgentID string `json:"agent_id"`
// Optional disambiguator; backend still verifies the share relation
AgentSourceTenantID uint64 `json:"agent_source_tenant_id,omitempty"`
// Browser source
LocalBrowserEnabled bool `json:"local_browser_enabled"`
// Whether web search is enabled for this request
WebSearchEnabled bool `json:"web_search_enabled"`
// Optional summary model ID for this request (overrides session default)
SummaryModelID string `json:"summary_model_id"`
// Per-request MCP services selected via @mention
MCPServiceIDs []string `json:"mcp_service_ids"`
// Per-request Skills selected via @mention
SkillNames []string `json:"skill_names"`
// @mentioned tag IDs (display/debug; scoped via MentionedItems)
TagIDs []string `json:"tag_ids"`
// @mentioned knowledge bases and files
MentionedItems []MentionedItemRequest `json:"mentioned_items"`
// Whether to disable auto title generation
DisableTitle bool `json:"disable_title"`
// Attached images for multimodal chat
Images []ImageAttachment `json:"images"`
// Attached files (documents, audio, etc.)
AttachmentUploads []AttachmentUpload `json:"attachment_uploads,omitempty"`
// Pre-uploaded session-scoped document IDs
AttachmentIDs []string `json:"attachment_ids,omitempty"`
// Source channel: "web", "api", "im", etc.
Channel string `json:"channel"`
SuggestionAttribution *types.SuggestionAttribution `json:"suggestion_attribution,omitempty"`
// QuestionOrigin is the knowledge source of a picked suggested question.
QuestionOrigin *types.QuestionOrigin `json:"question_origin,omitempty"`

View File

@ -2,8 +2,19 @@ package types
import (
"context"
"strings"
)
// UploadOnlyQuestion is the question asked on the user's behalf when a message
// carries an image or file but no text. The stored user message keeps its
// empty text; only model input uses this question.
func UploadOnlyQuestion(locale string) string {
if strings.HasPrefix(locale, "zh") {
return "请根据我上传的内容回答。"
}
return "Please answer based on what I uploaded."
}
// SteerMessageContent adds delivery context only to model input. The persisted
// user message and the UI always retain the user's original text.
func SteerMessageContent(content string) string {