fix(api): address resource_urls review follow-ups

Harden public URL mode: stop logging signed URLs at INFO, copy messages
before rewriting history responses, reuse winning grants on insert races,
sync the Go client resource_urls query param, and add knowledge-search
coverage.
main
wizardchen 2026-08-05 14:15:26 +08:00 committed by lyingbug
parent eb6ac0110c
commit e71c2dce2e
12 changed files with 274 additions and 22 deletions

View File

@ -98,6 +98,7 @@ DOCREADER_TRANSPORT=grpc
# 无法生成时该引用保持 resource:// 原样,客户端仍可回退到 /files。
# 注意:public 会为每个被引用文件签发限时(WeKnora 2 小时 / MinIO 24 小时)的匿名可读链接。
# 匿名的 embed 渠道与限定知识库的 API Key 不受本变量影响,始终返回 handle。
# 建议同时配置 SYSTEM_AES_KEY,以便复用 grant 行、稳定直链 URL 并降低读接口的写入压力。
# RESOURCE_URL_MODE=handle
# 对外暴露的 MCP Server 端口(默认 8082)。
# MCP_PORT=8082

View File

@ -9,6 +9,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
)
@ -77,8 +78,10 @@ func (c *Client) AgentQAStream(ctx context.Context, sessionID string, query stri
}
// AgentQAStreamWithRequest performs agent-based Q&A with SSE streaming using the full request payload.
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
func (c *Client) AgentQAStreamWithRequest(ctx context.Context,
sessionID string, request *AgentQARequest, callback AgentEventCallback,
opts ...ResourceURLOptions,
) error {
if request == nil {
return fmt.Errorf("agent QA request cannot be nil")
@ -88,7 +91,11 @@ func (c *Client) AgentQAStreamWithRequest(ctx context.Context,
}
path := fmt.Sprintf("/api/v1/agent-chat/%s", sessionID)
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil)
queryParams := url.Values{}
if len(opts) > 0 {
applyResourceURLQuery(queryParams, &opts[0])
}
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}

View File

@ -60,12 +60,15 @@ type MessageListResponse struct {
Data []Message `json:"data"`
}
// LoadMessages loads session messages, supports pagination and time filtering
// LoadMessages loads session messages, supports pagination and time filtering.
// Pass ResourceURLOptions to request public HTTP(S) file URLs instead of
// resource:// handles.
func (c *Client) LoadMessages(
ctx context.Context,
sessionID string,
limit int,
beforeTime *time.Time,
opts ...ResourceURLOptions,
) ([]Message, error) {
path := fmt.Sprintf("/api/v1/messages/%s/load", sessionID)
@ -75,6 +78,9 @@ func (c *Client) LoadMessages(
if beforeTime != nil {
queryParams.Add("before_time", beforeTime.Format(time.RFC3339Nano))
}
if len(opts) > 0 {
applyResourceURLQuery(queryParams, &opts[0])
}
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams)
if err != nil {

View File

@ -0,0 +1,30 @@
package client
import "net/url"
// ResourceURLMode selects how file references are returned in API responses.
// Empty leaves the choice to the server (handle by default, or
// RESOURCE_URL_MODE when set).
type ResourceURLMode string
const (
// ResourceURLModeHandle returns internal resource:// handles; clients fetch
// bytes through the authenticated /files proxy.
ResourceURLModeHandle ResourceURLMode = "handle"
// ResourceURLModePublic returns time-limited HTTP(S) URLs loadable without
// WeKnora credentials.
ResourceURLModePublic ResourceURLMode = "public"
)
// ResourceURLOptions carries the optional resource_urls query parameter shared
// by chat, message-history, and knowledge-search endpoints.
type ResourceURLOptions struct {
ResourceURLs ResourceURLMode
}
func applyResourceURLQuery(q url.Values, opts *ResourceURLOptions) {
if opts == nil || opts.ResourceURLs == "" || opts.ResourceURLs == ResourceURLModeHandle {
return
}
q.Set("resource_urls", string(opts.ResourceURLs))
}

View File

@ -0,0 +1,26 @@
package client
import (
"net/url"
"testing"
)
func TestApplyResourceURLQuery(t *testing.T) {
q := url.Values{}
applyResourceURLQuery(q, nil)
if len(q) != 0 {
t.Fatalf("nil opts: got %v, want empty", q)
}
q = url.Values{}
applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModeHandle})
if len(q) != 0 {
t.Fatalf("handle mode: got %v, want empty", q)
}
q = url.Values{}
applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModePublic})
if got := q.Get("resource_urls"); got != "public" {
t.Fatalf("public mode: got %q, want public", got)
}
}

View File

@ -262,17 +262,24 @@ type StreamResponse struct {
Data map[string]interface{} `json:"data,omitempty"` // Additional metadata for enhanced display
}
// KnowledgeQAStream knowledge Q&A streaming API
// KnowledgeQAStream knowledge Q&A streaming API.
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
func (c *Client) KnowledgeQAStream(
ctx context.Context,
sessionID string,
request *KnowledgeQARequest,
callback func(*StreamResponse) error,
opts ...ResourceURLOptions,
) error {
path := fmt.Sprintf("/api/v1/knowledge-chat/%s", sessionID)
debugLogger.Debug("knowledge_qa_stream_start", "session_id", sessionID, "query", request.Query)
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil)
queryParams := url.Values{}
if len(opts) > 0 {
applyResourceURLQuery(queryParams, &opts[0])
}
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams)
if err != nil {
debugLogger.Debug("request_failed", "error", err)
return err
@ -350,17 +357,22 @@ func (c *Client) KnowledgeQAStream(
return nil
}
// ContinueStream continues to receive an active stream for a session
// ContinueStream continues to receive an active stream for a session.
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
func (c *Client) ContinueStream(
ctx context.Context,
sessionID string,
messageID string,
callback func(*StreamResponse) error,
opts ...ResourceURLOptions,
) error {
path := fmt.Sprintf("/api/v1/sessions/continue-stream/%s", sessionID)
queryParams := url.Values{}
queryParams.Add("message_id", messageID)
if len(opts) > 0 {
applyResourceURLQuery(queryParams, &opts[0])
}
resp, err := c.doRequestStream(ctx, http.MethodGet, path, nil, queryParams)
if err != nil {
@ -463,15 +475,25 @@ type SearchKnowledgeResponse struct {
Data []*SearchResult `json:"data"`
}
// SearchKnowledge performs knowledge base search without LLM summarization
func (c *Client) SearchKnowledge(ctx context.Context, request *SearchKnowledgeRequest) ([]*SearchResult, error) {
// SearchKnowledge performs knowledge base search without LLM summarization.
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in results.
func (c *Client) SearchKnowledge(
ctx context.Context,
request *SearchKnowledgeRequest,
opts ...ResourceURLOptions,
) ([]*SearchResult, error) {
debugLogger.Debug("search_knowledge_start",
"knowledge_base_ids", request.KnowledgeBaseIDs,
"knowledge_ids", request.KnowledgeIDs,
"query", request.Query,
)
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, nil)
queryParams := url.Values{}
if len(opts) > 0 {
applyResourceURLQuery(queryParams, &opts[0])
}
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, queryParams)
if err != nil {
debugLogger.Debug("request_failed", "error", err)
return nil, err

View File

@ -7,6 +7,7 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
@ -14,6 +15,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"gorm.io/gorm"
)
const defaultResourceGrantTTL = 2 * time.Hour
@ -248,8 +250,14 @@ func (s *resourceCatalog) reuseOrCreateDerivedGrant(
case err == nil:
return token, nil
case isUniqueViolation(err):
// The row exists but the lookup above rejected it, so it is revoked:
// reviving it via a fresh insert would defeat the revocation.
// Another request may have won the race; a revoked row blocks re-insert.
winner, lookupErr := s.repo.GetValidGrant(ctx, tokenHash, now)
if lookupErr != nil {
return "", lookupErr
}
if winner != nil && winner.ResourceID == resourceID {
return token, nil
}
return "", nil
default:
return "", err
@ -276,10 +284,18 @@ func derivedGrantToken(resourceID string, ttl time.Duration) (string, time.Time,
return token, windowStart.Add(ttl), true
}
// isUniqueViolation reports whether err is a duplicate-key error. The repository
// returns driver errors verbatim, so the text is all we have.
// isUniqueViolation reports whether err is a duplicate-key error. Prefer
// gorm.ErrDuplicatedKey when TranslateError is enabled; fall back to the
// driver message for raw errors.
func isUniqueViolation(err error) bool {
return err != nil && strings.Contains(strings.ToLower(err.Error()), "unique")
if err == nil {
return false
}
if errors.Is(err, gorm.ErrDuplicatedKey) {
return true
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "duplicate") || strings.Contains(msg, "unique constraint")
}
func (s *resourceCatalog) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) {

View File

@ -131,10 +131,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) {
"Successfully retrieved recent messages, session ID: %s, message count: %d",
sessionID, len(messages),
)
rewriter.RewriteMessages(ctx, messages)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": messages,
"data": rewriter.RewriteMessagesResponse(ctx, messages),
})
return
}
@ -172,10 +171,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) {
"Successfully retrieved messages before time, session ID: %s, message count: %d",
sessionID, len(messages),
)
rewriter.RewriteMessages(ctx, messages)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": messages,
"data": rewriter.RewriteMessagesResponse(ctx, messages),
})
}

View File

@ -0,0 +1,100 @@
package session
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/Tencent/WeKnora/internal/middleware"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
type stubSearchSessionService struct {
interfaces.SessionService
}
func (s *stubSearchSessionService) SearchKnowledge(
_ context.Context, _ []string, _ []string, _ []types.TagScope, _ string,
) ([]*types.SearchResult, error) {
return []*types.SearchResult{{
Content: "chunk ![c](" + testResourceHandle + ")",
ImageInfo: `[{"url":"` + testResourceHandle + `"}]`,
}}, nil
}
func TestSearchKnowledge_PublicResourceURLs(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(middleware.ErrorHandler())
h := &Handler{
sessionService: &stubSearchSessionService{},
fileService: &stubResourceFileService{},
}
r.POST("/knowledge-search", h.SearchKnowledge)
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=public", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())
assert.NotContains(t, w.Body.String(), testResourceHandle)
assert.Contains(t, w.Body.String(), "cdn.example.com")
}
func TestSearchKnowledge_InvalidResourceURLMode(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(middleware.ErrorHandler())
h := &Handler{
sessionService: &stubSearchSessionService{},
fileService: &stubResourceFileService{},
}
r.POST("/knowledge-search", h.SearchKnowledge)
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=signed", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String())
assert.Contains(t, w.Body.String(), "resource_urls")
}
func TestSearchKnowledge_DefaultKeepsHandles(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(middleware.ErrorHandler())
h := &Handler{
sessionService: &stubSearchSessionService{},
fileService: &stubResourceFileService{},
}
r.POST("/knowledge-search", h.SearchKnowledge)
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
req := httptest.NewRequest(http.MethodPost, "/knowledge-search", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())
var resp struct {
Data []struct {
Content string `json:"content"`
} `json:"data"`
}
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
require.Len(t, resp.Data, 1)
assert.Contains(t, resp.Data[0].Content, testResourceHandle)
}

View File

@ -2,6 +2,7 @@ package storageurl
import (
"context"
"encoding/json"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@ -37,6 +38,31 @@ func NewRequestRewriter(
return NewRewriter(resolver, "API")
}
// RewriteMessagesResponse returns rewritten message copies so callers can keep
// the originals untouched (for example objects returned from a service cache).
func (w *Rewriter) RewriteMessagesResponse(ctx context.Context, messages []*types.Message) []*types.Message {
if !w.Enabled() || len(messages) == 0 {
return messages
}
out := cloneMessages(messages)
w.RewriteMessages(ctx, out)
return out
}
// cloneMessages deep-copies messages via JSON so nested slices and agent steps
// are independent of the source slice.
func cloneMessages(messages []*types.Message) []*types.Message {
data, err := json.Marshal(messages)
if err != nil {
return messages
}
var out []*types.Message
if err := json.Unmarshal(data, &out); err != nil {
return messages
}
return out
}
// RewriteMessages replaces storage references in a message history response so
// clients receive loadable image URLs. Messages are mutated in place.
func (w *Rewriter) RewriteMessages(ctx context.Context, messages []*types.Message) {

View File

@ -71,6 +71,26 @@ func TestRewriteMessages_DisabledLeavesHandles(t *testing.T) {
assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", messages[0].Content)
}
func TestRewriteMessagesResponse_DoesNotMutateOriginals(t *testing.T) {
w := publicRewriter("https://cdn.example.com/x.png")
original := &types.Message{
Content: "![a](resource://xifDo7NTSL300Lp1goVutw)",
KnowledgeReferences: types.References{{
Content: "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)",
}},
}
messages := []*types.Message{original}
out := w.RewriteMessagesResponse(context.Background(), messages)
require.Len(t, out, 1)
assert.NotSame(t, original, out[0])
assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", original.Content)
assert.Equal(t, "![a](https://cdn.example.com/x.png)", out[0].Content)
assert.Equal(t, "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)", original.KnowledgeReferences[0].Content)
assert.Equal(t, "chunk ![c](https://cdn.example.com/x.png)", out[0].KnowledgeReferences[0].Content)
}
// SSE references payloads share their *SearchResult pointers with the stream
// replay buffer and the assistant message being persisted, so rewriting must not
// mutate the originals.

View File

@ -62,10 +62,9 @@ type Resolver interface {
// a signature.
//
// Logging policy:
// - A successful rewrite logs at INFO with the full signed URL so operators
// can copy it out of logs and verify public reachability directly. The
// trade-off: anyone with log access can use a signed URL until it expires
// (WeKnora grants 2h, MinIO 24h). Acceptable for diagnosability.
// - A successful rewrite logs the source reference at INFO. The signed URL
// is DEBUG-only so log aggregation cannot hand out anonymously readable
// links; operators can raise the log level when verifying reachability.
// - Failure or no-op logs at WARN. The no-op case usually means
// APP_EXTERNAL_URL is unset, the most common cause of "image broken in the
// IM channel / in my app" reports.
@ -150,7 +149,8 @@ func (w *Rewriter) resolve(ctx context.Context, ref string) string {
w.logPrefix, httpURL, ref)
return ref
}
logger.Infof(ctx, "[%s] storage URL rewrite: src=%s dst=%s", w.logPrefix, ref, httpURL)
logger.Infof(ctx, "[%s] storage URL rewrite: src=%s", w.logPrefix, ref)
logger.Debugf(ctx, "[%s] storage URL rewrite dst=%s", w.logPrefix, httpURL)
return httpURL
}