From e71c2dce2eb5a9c28809af8801c18fb3cd9e3c45 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 5 Aug 2026 14:15:26 +0800 Subject: [PATCH] fix(api): address resource_urls review follow-ups Harden public URL mode: stop logging signed URLs at INFO, copy messages before rewriting history responses, reuse winning grants on insert races, sync the Go client resource_urls query param, and add knowledge-search coverage. --- .env.example | 1 + client/agent.go | 9 +- client/message.go | 8 +- client/resource_urls.go | 30 ++++++ client/resource_urls_test.go | 26 +++++ client/session.go | 34 ++++-- internal/application/service/resource.go | 26 ++++- internal/handler/message.go | 6 +- .../search_knowledge_resource_urls_test.go | 100 ++++++++++++++++++ internal/storageurl/request.go | 26 +++++ internal/storageurl/request_test.go | 20 ++++ internal/storageurl/storageurl.go | 10 +- 12 files changed, 274 insertions(+), 22 deletions(-) create mode 100644 client/resource_urls.go create mode 100644 client/resource_urls_test.go create mode 100644 internal/handler/session/search_knowledge_resource_urls_test.go diff --git a/.env.example b/.env.example index f31f5b08..776260f4 100644 --- a/.env.example +++ b/.env.example @@ -98,6 +98,7 @@ DOCREADER_TRANSPORT=grpc # 无法生成时该引用保持 resource:// 原样,客户端仍可回退到 /files。 # 注意:public 会为每个被引用文件签发限时(WeKnora 2 小时 / MinIO 24 小时)的匿名可读链接。 # 匿名的 embed 渠道与限定知识库的 API Key 不受本变量影响,始终返回 handle。 +# 建议同时配置 SYSTEM_AES_KEY,以便复用 grant 行、稳定直链 URL 并降低读接口的写入压力。 # RESOURCE_URL_MODE=handle # 对外暴露的 MCP Server 端口(默认 8082)。 # MCP_PORT=8082 diff --git a/client/agent.go b/client/agent.go index 0f560cde..ccc9ee84 100644 --- a/client/agent.go +++ b/client/agent.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" ) @@ -77,8 +78,10 @@ func (c *Client) AgentQAStream(ctx context.Context, sessionID string, query stri } // AgentQAStreamWithRequest performs agent-based Q&A with SSE streaming using the full request payload. +// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream. func (c *Client) AgentQAStreamWithRequest(ctx context.Context, sessionID string, request *AgentQARequest, callback AgentEventCallback, + opts ...ResourceURLOptions, ) error { if request == nil { return fmt.Errorf("agent QA request cannot be nil") @@ -88,7 +91,11 @@ func (c *Client) AgentQAStreamWithRequest(ctx context.Context, } path := fmt.Sprintf("/api/v1/agent-chat/%s", sessionID) - resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil) + queryParams := url.Values{} + if len(opts) > 0 { + applyResourceURLQuery(queryParams, &opts[0]) + } + resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams) if err != nil { return fmt.Errorf("request failed: %w", err) } diff --git a/client/message.go b/client/message.go index b5c59e3c..b0a7df8d 100644 --- a/client/message.go +++ b/client/message.go @@ -60,12 +60,15 @@ type MessageListResponse struct { Data []Message `json:"data"` } -// LoadMessages loads session messages, supports pagination and time filtering +// LoadMessages loads session messages, supports pagination and time filtering. +// Pass ResourceURLOptions to request public HTTP(S) file URLs instead of +// resource:// handles. func (c *Client) LoadMessages( ctx context.Context, sessionID string, limit int, beforeTime *time.Time, + opts ...ResourceURLOptions, ) ([]Message, error) { path := fmt.Sprintf("/api/v1/messages/%s/load", sessionID) @@ -75,6 +78,9 @@ func (c *Client) LoadMessages( if beforeTime != nil { queryParams.Add("before_time", beforeTime.Format(time.RFC3339Nano)) } + if len(opts) > 0 { + applyResourceURLQuery(queryParams, &opts[0]) + } resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) if err != nil { diff --git a/client/resource_urls.go b/client/resource_urls.go new file mode 100644 index 00000000..442fae48 --- /dev/null +++ b/client/resource_urls.go @@ -0,0 +1,30 @@ +package client + +import "net/url" + +// ResourceURLMode selects how file references are returned in API responses. +// Empty leaves the choice to the server (handle by default, or +// RESOURCE_URL_MODE when set). +type ResourceURLMode string + +const ( + // ResourceURLModeHandle returns internal resource:// handles; clients fetch + // bytes through the authenticated /files proxy. + ResourceURLModeHandle ResourceURLMode = "handle" + // ResourceURLModePublic returns time-limited HTTP(S) URLs loadable without + // WeKnora credentials. + ResourceURLModePublic ResourceURLMode = "public" +) + +// ResourceURLOptions carries the optional resource_urls query parameter shared +// by chat, message-history, and knowledge-search endpoints. +type ResourceURLOptions struct { + ResourceURLs ResourceURLMode +} + +func applyResourceURLQuery(q url.Values, opts *ResourceURLOptions) { + if opts == nil || opts.ResourceURLs == "" || opts.ResourceURLs == ResourceURLModeHandle { + return + } + q.Set("resource_urls", string(opts.ResourceURLs)) +} diff --git a/client/resource_urls_test.go b/client/resource_urls_test.go new file mode 100644 index 00000000..f081bd75 --- /dev/null +++ b/client/resource_urls_test.go @@ -0,0 +1,26 @@ +package client + +import ( + "net/url" + "testing" +) + +func TestApplyResourceURLQuery(t *testing.T) { + q := url.Values{} + applyResourceURLQuery(q, nil) + if len(q) != 0 { + t.Fatalf("nil opts: got %v, want empty", q) + } + + q = url.Values{} + applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModeHandle}) + if len(q) != 0 { + t.Fatalf("handle mode: got %v, want empty", q) + } + + q = url.Values{} + applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModePublic}) + if got := q.Get("resource_urls"); got != "public" { + t.Fatalf("public mode: got %q, want public", got) + } +} diff --git a/client/session.go b/client/session.go index 37ff2ecc..31863cea 100644 --- a/client/session.go +++ b/client/session.go @@ -262,17 +262,24 @@ type StreamResponse struct { Data map[string]interface{} `json:"data,omitempty"` // Additional metadata for enhanced display } -// KnowledgeQAStream knowledge Q&A streaming API +// KnowledgeQAStream knowledge Q&A streaming API. +// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream. func (c *Client) KnowledgeQAStream( ctx context.Context, sessionID string, request *KnowledgeQARequest, callback func(*StreamResponse) error, + opts ...ResourceURLOptions, ) error { path := fmt.Sprintf("/api/v1/knowledge-chat/%s", sessionID) debugLogger.Debug("knowledge_qa_stream_start", "session_id", sessionID, "query", request.Query) - resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil) + queryParams := url.Values{} + if len(opts) > 0 { + applyResourceURLQuery(queryParams, &opts[0]) + } + + resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams) if err != nil { debugLogger.Debug("request_failed", "error", err) return err @@ -350,17 +357,22 @@ func (c *Client) KnowledgeQAStream( return nil } -// ContinueStream continues to receive an active stream for a session +// ContinueStream continues to receive an active stream for a session. +// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream. func (c *Client) ContinueStream( ctx context.Context, sessionID string, messageID string, callback func(*StreamResponse) error, + opts ...ResourceURLOptions, ) error { path := fmt.Sprintf("/api/v1/sessions/continue-stream/%s", sessionID) queryParams := url.Values{} queryParams.Add("message_id", messageID) + if len(opts) > 0 { + applyResourceURLQuery(queryParams, &opts[0]) + } resp, err := c.doRequestStream(ctx, http.MethodGet, path, nil, queryParams) if err != nil { @@ -463,15 +475,25 @@ type SearchKnowledgeResponse struct { Data []*SearchResult `json:"data"` } -// SearchKnowledge performs knowledge base search without LLM summarization -func (c *Client) SearchKnowledge(ctx context.Context, request *SearchKnowledgeRequest) ([]*SearchResult, error) { +// SearchKnowledge performs knowledge base search without LLM summarization. +// Pass ResourceURLOptions to receive public HTTP(S) file URLs in results. +func (c *Client) SearchKnowledge( + ctx context.Context, + request *SearchKnowledgeRequest, + opts ...ResourceURLOptions, +) ([]*SearchResult, error) { debugLogger.Debug("search_knowledge_start", "knowledge_base_ids", request.KnowledgeBaseIDs, "knowledge_ids", request.KnowledgeIDs, "query", request.Query, ) - resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, nil) + queryParams := url.Values{} + if len(opts) > 0 { + applyResourceURLQuery(queryParams, &opts[0]) + } + + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, queryParams) if err != nil { debugLogger.Debug("request_failed", "error", err) return nil, err diff --git a/internal/application/service/resource.go b/internal/application/service/resource.go index 63950866..50479a9c 100644 --- a/internal/application/service/resource.go +++ b/internal/application/service/resource.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "strings" "time" @@ -14,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" + "gorm.io/gorm" ) const defaultResourceGrantTTL = 2 * time.Hour @@ -248,8 +250,14 @@ func (s *resourceCatalog) reuseOrCreateDerivedGrant( case err == nil: return token, nil case isUniqueViolation(err): - // The row exists but the lookup above rejected it, so it is revoked: - // reviving it via a fresh insert would defeat the revocation. + // Another request may have won the race; a revoked row blocks re-insert. + winner, lookupErr := s.repo.GetValidGrant(ctx, tokenHash, now) + if lookupErr != nil { + return "", lookupErr + } + if winner != nil && winner.ResourceID == resourceID { + return token, nil + } return "", nil default: return "", err @@ -276,10 +284,18 @@ func derivedGrantToken(resourceID string, ttl time.Duration) (string, time.Time, return token, windowStart.Add(ttl), true } -// isUniqueViolation reports whether err is a duplicate-key error. The repository -// returns driver errors verbatim, so the text is all we have. +// isUniqueViolation reports whether err is a duplicate-key error. Prefer +// gorm.ErrDuplicatedKey when TranslateError is enabled; fall back to the +// driver message for raw errors. func isUniqueViolation(err error) bool { - return err != nil && strings.Contains(strings.ToLower(err.Error()), "unique") + if err == nil { + return false + } + if errors.Is(err, gorm.ErrDuplicatedKey) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "duplicate") || strings.Contains(msg, "unique constraint") } func (s *resourceCatalog) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) { diff --git a/internal/handler/message.go b/internal/handler/message.go index 90fc16fa..991bcbc5 100644 --- a/internal/handler/message.go +++ b/internal/handler/message.go @@ -131,10 +131,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) { "Successfully retrieved recent messages, session ID: %s, message count: %d", sessionID, len(messages), ) - rewriter.RewriteMessages(ctx, messages) c.JSON(http.StatusOK, gin.H{ "success": true, - "data": messages, + "data": rewriter.RewriteMessagesResponse(ctx, messages), }) return } @@ -172,10 +171,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) { "Successfully retrieved messages before time, session ID: %s, message count: %d", sessionID, len(messages), ) - rewriter.RewriteMessages(ctx, messages) c.JSON(http.StatusOK, gin.H{ "success": true, - "data": messages, + "data": rewriter.RewriteMessagesResponse(ctx, messages), }) } diff --git a/internal/handler/session/search_knowledge_resource_urls_test.go b/internal/handler/session/search_knowledge_resource_urls_test.go new file mode 100644 index 00000000..b114042d --- /dev/null +++ b/internal/handler/session/search_knowledge_resource_urls_test.go @@ -0,0 +1,100 @@ +package session + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +type stubSearchSessionService struct { + interfaces.SessionService +} + +func (s *stubSearchSessionService) SearchKnowledge( + _ context.Context, _ []string, _ []string, _ []types.TagScope, _ string, +) ([]*types.SearchResult, error) { + return []*types.SearchResult{{ + Content: "chunk ![c](" + testResourceHandle + ")", + ImageInfo: `[{"url":"` + testResourceHandle + `"}]`, + }}, nil +} + +func TestSearchKnowledge_PublicResourceURLs(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(middleware.ErrorHandler()) + h := &Handler{ + sessionService: &stubSearchSessionService{}, + fileService: &stubResourceFileService{}, + } + r.POST("/knowledge-search", h.SearchKnowledge) + + body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`) + req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=public", body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.NotContains(t, w.Body.String(), testResourceHandle) + assert.Contains(t, w.Body.String(), "cdn.example.com") +} + +func TestSearchKnowledge_InvalidResourceURLMode(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(middleware.ErrorHandler()) + h := &Handler{ + sessionService: &stubSearchSessionService{}, + fileService: &stubResourceFileService{}, + } + r.POST("/knowledge-search", h.SearchKnowledge) + + body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`) + req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=signed", body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + assert.Contains(t, w.Body.String(), "resource_urls") +} + +func TestSearchKnowledge_DefaultKeepsHandles(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(middleware.ErrorHandler()) + h := &Handler{ + sessionService: &stubSearchSessionService{}, + fileService: &stubResourceFileService{}, + } + r.POST("/knowledge-search", h.SearchKnowledge) + + body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`) + req := httptest.NewRequest(http.MethodPost, "/knowledge-search", body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + + var resp struct { + Data []struct { + Content string `json:"content"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.Len(t, resp.Data, 1) + assert.Contains(t, resp.Data[0].Content, testResourceHandle) +} diff --git a/internal/storageurl/request.go b/internal/storageurl/request.go index dc5039fc..b7275e03 100644 --- a/internal/storageurl/request.go +++ b/internal/storageurl/request.go @@ -2,6 +2,7 @@ package storageurl import ( "context" + "encoding/json" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -37,6 +38,31 @@ func NewRequestRewriter( return NewRewriter(resolver, "API") } +// RewriteMessagesResponse returns rewritten message copies so callers can keep +// the originals untouched (for example objects returned from a service cache). +func (w *Rewriter) RewriteMessagesResponse(ctx context.Context, messages []*types.Message) []*types.Message { + if !w.Enabled() || len(messages) == 0 { + return messages + } + out := cloneMessages(messages) + w.RewriteMessages(ctx, out) + return out +} + +// cloneMessages deep-copies messages via JSON so nested slices and agent steps +// are independent of the source slice. +func cloneMessages(messages []*types.Message) []*types.Message { + data, err := json.Marshal(messages) + if err != nil { + return messages + } + var out []*types.Message + if err := json.Unmarshal(data, &out); err != nil { + return messages + } + return out +} + // RewriteMessages replaces storage references in a message history response so // clients receive loadable image URLs. Messages are mutated in place. func (w *Rewriter) RewriteMessages(ctx context.Context, messages []*types.Message) { diff --git a/internal/storageurl/request_test.go b/internal/storageurl/request_test.go index 96bcd85a..2008fa20 100644 --- a/internal/storageurl/request_test.go +++ b/internal/storageurl/request_test.go @@ -71,6 +71,26 @@ func TestRewriteMessages_DisabledLeavesHandles(t *testing.T) { assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", messages[0].Content) } +func TestRewriteMessagesResponse_DoesNotMutateOriginals(t *testing.T) { + w := publicRewriter("https://cdn.example.com/x.png") + original := &types.Message{ + Content: "![a](resource://xifDo7NTSL300Lp1goVutw)", + KnowledgeReferences: types.References{{ + Content: "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)", + }}, + } + messages := []*types.Message{original} + + out := w.RewriteMessagesResponse(context.Background(), messages) + + require.Len(t, out, 1) + assert.NotSame(t, original, out[0]) + assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", original.Content) + assert.Equal(t, "![a](https://cdn.example.com/x.png)", out[0].Content) + assert.Equal(t, "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)", original.KnowledgeReferences[0].Content) + assert.Equal(t, "chunk ![c](https://cdn.example.com/x.png)", out[0].KnowledgeReferences[0].Content) +} + // SSE references payloads share their *SearchResult pointers with the stream // replay buffer and the assistant message being persisted, so rewriting must not // mutate the originals. diff --git a/internal/storageurl/storageurl.go b/internal/storageurl/storageurl.go index 3ebf2dda..86926876 100644 --- a/internal/storageurl/storageurl.go +++ b/internal/storageurl/storageurl.go @@ -62,10 +62,9 @@ type Resolver interface { // a signature. // // Logging policy: -// - A successful rewrite logs at INFO with the full signed URL so operators -// can copy it out of logs and verify public reachability directly. The -// trade-off: anyone with log access can use a signed URL until it expires -// (WeKnora grants 2h, MinIO 24h). Acceptable for diagnosability. +// - A successful rewrite logs the source reference at INFO. The signed URL +// is DEBUG-only so log aggregation cannot hand out anonymously readable +// links; operators can raise the log level when verifying reachability. // - Failure or no-op logs at WARN. The no-op case usually means // APP_EXTERNAL_URL is unset, the most common cause of "image broken in the // IM channel / in my app" reports. @@ -150,7 +149,8 @@ func (w *Rewriter) resolve(ctx context.Context, ref string) string { w.logPrefix, httpURL, ref) return ref } - logger.Infof(ctx, "[%s] storage URL rewrite: src=%s dst=%s", w.logPrefix, ref, httpURL) + logger.Infof(ctx, "[%s] storage URL rewrite: src=%s", w.logPrefix, ref) + logger.Debugf(ctx, "[%s] storage URL rewrite dst=%s", w.logPrefix, httpURL) return httpURL }