fix(api): address resource_urls review follow-ups
Harden public URL mode: stop logging signed URLs at INFO, copy messages before rewriting history responses, reuse winning grants on insert races, sync the Go client resource_urls query param, and add knowledge-search coverage.main
parent
eb6ac0110c
commit
e71c2dce2e
|
|
@ -98,6 +98,7 @@ DOCREADER_TRANSPORT=grpc
|
|||
# 无法生成时该引用保持 resource:// 原样,客户端仍可回退到 /files。
|
||||
# 注意:public 会为每个被引用文件签发限时(WeKnora 2 小时 / MinIO 24 小时)的匿名可读链接。
|
||||
# 匿名的 embed 渠道与限定知识库的 API Key 不受本变量影响,始终返回 handle。
|
||||
# 建议同时配置 SYSTEM_AES_KEY,以便复用 grant 行、稳定直链 URL 并降低读接口的写入压力。
|
||||
# RESOURCE_URL_MODE=handle
|
||||
# 对外暴露的 MCP Server 端口(默认 8082)。
|
||||
# MCP_PORT=8082
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
@ -77,8 +78,10 @@ func (c *Client) AgentQAStream(ctx context.Context, sessionID string, query stri
|
|||
}
|
||||
|
||||
// AgentQAStreamWithRequest performs agent-based Q&A with SSE streaming using the full request payload.
|
||||
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
|
||||
func (c *Client) AgentQAStreamWithRequest(ctx context.Context,
|
||||
sessionID string, request *AgentQARequest, callback AgentEventCallback,
|
||||
opts ...ResourceURLOptions,
|
||||
) error {
|
||||
if request == nil {
|
||||
return fmt.Errorf("agent QA request cannot be nil")
|
||||
|
|
@ -88,7 +91,11 @@ func (c *Client) AgentQAStreamWithRequest(ctx context.Context,
|
|||
}
|
||||
|
||||
path := fmt.Sprintf("/api/v1/agent-chat/%s", sessionID)
|
||||
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil)
|
||||
queryParams := url.Values{}
|
||||
if len(opts) > 0 {
|
||||
applyResourceURLQuery(queryParams, &opts[0])
|
||||
}
|
||||
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,12 +60,15 @@ type MessageListResponse struct {
|
|||
Data []Message `json:"data"`
|
||||
}
|
||||
|
||||
// LoadMessages loads session messages, supports pagination and time filtering
|
||||
// LoadMessages loads session messages, supports pagination and time filtering.
|
||||
// Pass ResourceURLOptions to request public HTTP(S) file URLs instead of
|
||||
// resource:// handles.
|
||||
func (c *Client) LoadMessages(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
limit int,
|
||||
beforeTime *time.Time,
|
||||
opts ...ResourceURLOptions,
|
||||
) ([]Message, error) {
|
||||
path := fmt.Sprintf("/api/v1/messages/%s/load", sessionID)
|
||||
|
||||
|
|
@ -75,6 +78,9 @@ func (c *Client) LoadMessages(
|
|||
if beforeTime != nil {
|
||||
queryParams.Add("before_time", beforeTime.Format(time.RFC3339Nano))
|
||||
}
|
||||
if len(opts) > 0 {
|
||||
applyResourceURLQuery(queryParams, &opts[0])
|
||||
}
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
package client
|
||||
|
||||
import "net/url"
|
||||
|
||||
// ResourceURLMode selects how file references are returned in API responses.
|
||||
// Empty leaves the choice to the server (handle by default, or
|
||||
// RESOURCE_URL_MODE when set).
|
||||
type ResourceURLMode string
|
||||
|
||||
const (
|
||||
// ResourceURLModeHandle returns internal resource:// handles; clients fetch
|
||||
// bytes through the authenticated /files proxy.
|
||||
ResourceURLModeHandle ResourceURLMode = "handle"
|
||||
// ResourceURLModePublic returns time-limited HTTP(S) URLs loadable without
|
||||
// WeKnora credentials.
|
||||
ResourceURLModePublic ResourceURLMode = "public"
|
||||
)
|
||||
|
||||
// ResourceURLOptions carries the optional resource_urls query parameter shared
|
||||
// by chat, message-history, and knowledge-search endpoints.
|
||||
type ResourceURLOptions struct {
|
||||
ResourceURLs ResourceURLMode
|
||||
}
|
||||
|
||||
func applyResourceURLQuery(q url.Values, opts *ResourceURLOptions) {
|
||||
if opts == nil || opts.ResourceURLs == "" || opts.ResourceURLs == ResourceURLModeHandle {
|
||||
return
|
||||
}
|
||||
q.Set("resource_urls", string(opts.ResourceURLs))
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package client
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestApplyResourceURLQuery(t *testing.T) {
|
||||
q := url.Values{}
|
||||
applyResourceURLQuery(q, nil)
|
||||
if len(q) != 0 {
|
||||
t.Fatalf("nil opts: got %v, want empty", q)
|
||||
}
|
||||
|
||||
q = url.Values{}
|
||||
applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModeHandle})
|
||||
if len(q) != 0 {
|
||||
t.Fatalf("handle mode: got %v, want empty", q)
|
||||
}
|
||||
|
||||
q = url.Values{}
|
||||
applyResourceURLQuery(q, &ResourceURLOptions{ResourceURLs: ResourceURLModePublic})
|
||||
if got := q.Get("resource_urls"); got != "public" {
|
||||
t.Fatalf("public mode: got %q, want public", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -262,17 +262,24 @@ type StreamResponse struct {
|
|||
Data map[string]interface{} `json:"data,omitempty"` // Additional metadata for enhanced display
|
||||
}
|
||||
|
||||
// KnowledgeQAStream knowledge Q&A streaming API
|
||||
// KnowledgeQAStream knowledge Q&A streaming API.
|
||||
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
|
||||
func (c *Client) KnowledgeQAStream(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
request *KnowledgeQARequest,
|
||||
callback func(*StreamResponse) error,
|
||||
opts ...ResourceURLOptions,
|
||||
) error {
|
||||
path := fmt.Sprintf("/api/v1/knowledge-chat/%s", sessionID)
|
||||
debugLogger.Debug("knowledge_qa_stream_start", "session_id", sessionID, "query", request.Query)
|
||||
|
||||
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, nil)
|
||||
queryParams := url.Values{}
|
||||
if len(opts) > 0 {
|
||||
applyResourceURLQuery(queryParams, &opts[0])
|
||||
}
|
||||
|
||||
resp, err := c.doRequestStream(ctx, http.MethodPost, path, request, queryParams)
|
||||
if err != nil {
|
||||
debugLogger.Debug("request_failed", "error", err)
|
||||
return err
|
||||
|
|
@ -350,17 +357,22 @@ func (c *Client) KnowledgeQAStream(
|
|||
return nil
|
||||
}
|
||||
|
||||
// ContinueStream continues to receive an active stream for a session
|
||||
// ContinueStream continues to receive an active stream for a session.
|
||||
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in the stream.
|
||||
func (c *Client) ContinueStream(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
messageID string,
|
||||
callback func(*StreamResponse) error,
|
||||
opts ...ResourceURLOptions,
|
||||
) error {
|
||||
path := fmt.Sprintf("/api/v1/sessions/continue-stream/%s", sessionID)
|
||||
|
||||
queryParams := url.Values{}
|
||||
queryParams.Add("message_id", messageID)
|
||||
if len(opts) > 0 {
|
||||
applyResourceURLQuery(queryParams, &opts[0])
|
||||
}
|
||||
|
||||
resp, err := c.doRequestStream(ctx, http.MethodGet, path, nil, queryParams)
|
||||
if err != nil {
|
||||
|
|
@ -463,15 +475,25 @@ type SearchKnowledgeResponse struct {
|
|||
Data []*SearchResult `json:"data"`
|
||||
}
|
||||
|
||||
// SearchKnowledge performs knowledge base search without LLM summarization
|
||||
func (c *Client) SearchKnowledge(ctx context.Context, request *SearchKnowledgeRequest) ([]*SearchResult, error) {
|
||||
// SearchKnowledge performs knowledge base search without LLM summarization.
|
||||
// Pass ResourceURLOptions to receive public HTTP(S) file URLs in results.
|
||||
func (c *Client) SearchKnowledge(
|
||||
ctx context.Context,
|
||||
request *SearchKnowledgeRequest,
|
||||
opts ...ResourceURLOptions,
|
||||
) ([]*SearchResult, error) {
|
||||
debugLogger.Debug("search_knowledge_start",
|
||||
"knowledge_base_ids", request.KnowledgeBaseIDs,
|
||||
"knowledge_ids", request.KnowledgeIDs,
|
||||
"query", request.Query,
|
||||
)
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, nil)
|
||||
queryParams := url.Values{}
|
||||
if len(opts) > 0 {
|
||||
applyResourceURLQuery(queryParams, &opts[0])
|
||||
}
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, queryParams)
|
||||
if err != nil {
|
||||
debugLogger.Debug("request_failed", "error", err)
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -14,6 +15,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const defaultResourceGrantTTL = 2 * time.Hour
|
||||
|
|
@ -248,8 +250,14 @@ func (s *resourceCatalog) reuseOrCreateDerivedGrant(
|
|||
case err == nil:
|
||||
return token, nil
|
||||
case isUniqueViolation(err):
|
||||
// The row exists but the lookup above rejected it, so it is revoked:
|
||||
// reviving it via a fresh insert would defeat the revocation.
|
||||
// Another request may have won the race; a revoked row blocks re-insert.
|
||||
winner, lookupErr := s.repo.GetValidGrant(ctx, tokenHash, now)
|
||||
if lookupErr != nil {
|
||||
return "", lookupErr
|
||||
}
|
||||
if winner != nil && winner.ResourceID == resourceID {
|
||||
return token, nil
|
||||
}
|
||||
return "", nil
|
||||
default:
|
||||
return "", err
|
||||
|
|
@ -276,10 +284,18 @@ func derivedGrantToken(resourceID string, ttl time.Duration) (string, time.Time,
|
|||
return token, windowStart.Add(ttl), true
|
||||
}
|
||||
|
||||
// isUniqueViolation reports whether err is a duplicate-key error. The repository
|
||||
// returns driver errors verbatim, so the text is all we have.
|
||||
// isUniqueViolation reports whether err is a duplicate-key error. Prefer
|
||||
// gorm.ErrDuplicatedKey when TranslateError is enabled; fall back to the
|
||||
// driver message for raw errors.
|
||||
func isUniqueViolation(err error) bool {
|
||||
return err != nil && strings.Contains(strings.ToLower(err.Error()), "unique")
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, gorm.ErrDuplicatedKey) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "duplicate") || strings.Contains(msg, "unique constraint")
|
||||
}
|
||||
|
||||
func (s *resourceCatalog) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) {
|
||||
|
|
|
|||
|
|
@ -131,10 +131,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) {
|
|||
"Successfully retrieved recent messages, session ID: %s, message count: %d",
|
||||
sessionID, len(messages),
|
||||
)
|
||||
rewriter.RewriteMessages(ctx, messages)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": messages,
|
||||
"data": rewriter.RewriteMessagesResponse(ctx, messages),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -172,10 +171,9 @@ func (h *MessageHandler) LoadMessages(c *gin.Context) {
|
|||
"Successfully retrieved messages before time, session ID: %s, message count: %d",
|
||||
sessionID, len(messages),
|
||||
)
|
||||
rewriter.RewriteMessages(ctx, messages)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": messages,
|
||||
"data": rewriter.RewriteMessagesResponse(ctx, messages),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
package session
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/middleware"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
type stubSearchSessionService struct {
|
||||
interfaces.SessionService
|
||||
}
|
||||
|
||||
func (s *stubSearchSessionService) SearchKnowledge(
|
||||
_ context.Context, _ []string, _ []string, _ []types.TagScope, _ string,
|
||||
) ([]*types.SearchResult, error) {
|
||||
return []*types.SearchResult{{
|
||||
Content: "chunk ",
|
||||
ImageInfo: `[{"url":"` + testResourceHandle + `"}]`,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func TestSearchKnowledge_PublicResourceURLs(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
r := gin.New()
|
||||
r.Use(middleware.ErrorHandler())
|
||||
h := &Handler{
|
||||
sessionService: &stubSearchSessionService{},
|
||||
fileService: &stubResourceFileService{},
|
||||
}
|
||||
r.POST("/knowledge-search", h.SearchKnowledge)
|
||||
|
||||
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=public", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())
|
||||
assert.NotContains(t, w.Body.String(), testResourceHandle)
|
||||
assert.Contains(t, w.Body.String(), "cdn.example.com")
|
||||
}
|
||||
|
||||
func TestSearchKnowledge_InvalidResourceURLMode(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
r := gin.New()
|
||||
r.Use(middleware.ErrorHandler())
|
||||
h := &Handler{
|
||||
sessionService: &stubSearchSessionService{},
|
||||
fileService: &stubResourceFileService{},
|
||||
}
|
||||
r.POST("/knowledge-search", h.SearchKnowledge)
|
||||
|
||||
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/knowledge-search?resource_urls=signed", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String())
|
||||
assert.Contains(t, w.Body.String(), "resource_urls")
|
||||
}
|
||||
|
||||
func TestSearchKnowledge_DefaultKeepsHandles(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
r := gin.New()
|
||||
r.Use(middleware.ErrorHandler())
|
||||
h := &Handler{
|
||||
sessionService: &stubSearchSessionService{},
|
||||
fileService: &stubResourceFileService{},
|
||||
}
|
||||
r.POST("/knowledge-search", h.SearchKnowledge)
|
||||
|
||||
body := bytes.NewBufferString(`{"query":"diagram","knowledge_base_ids":["kb-1"]}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/knowledge-search", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())
|
||||
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"data"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
require.Len(t, resp.Data, 1)
|
||||
assert.Contains(t, resp.Data[0].Content, testResourceHandle)
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package storageurl
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
|
|
@ -37,6 +38,31 @@ func NewRequestRewriter(
|
|||
return NewRewriter(resolver, "API")
|
||||
}
|
||||
|
||||
// RewriteMessagesResponse returns rewritten message copies so callers can keep
|
||||
// the originals untouched (for example objects returned from a service cache).
|
||||
func (w *Rewriter) RewriteMessagesResponse(ctx context.Context, messages []*types.Message) []*types.Message {
|
||||
if !w.Enabled() || len(messages) == 0 {
|
||||
return messages
|
||||
}
|
||||
out := cloneMessages(messages)
|
||||
w.RewriteMessages(ctx, out)
|
||||
return out
|
||||
}
|
||||
|
||||
// cloneMessages deep-copies messages via JSON so nested slices and agent steps
|
||||
// are independent of the source slice.
|
||||
func cloneMessages(messages []*types.Message) []*types.Message {
|
||||
data, err := json.Marshal(messages)
|
||||
if err != nil {
|
||||
return messages
|
||||
}
|
||||
var out []*types.Message
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return messages
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RewriteMessages replaces storage references in a message history response so
|
||||
// clients receive loadable image URLs. Messages are mutated in place.
|
||||
func (w *Rewriter) RewriteMessages(ctx context.Context, messages []*types.Message) {
|
||||
|
|
|
|||
|
|
@ -71,6 +71,26 @@ func TestRewriteMessages_DisabledLeavesHandles(t *testing.T) {
|
|||
assert.Equal(t, "", messages[0].Content)
|
||||
}
|
||||
|
||||
func TestRewriteMessagesResponse_DoesNotMutateOriginals(t *testing.T) {
|
||||
w := publicRewriter("https://cdn.example.com/x.png")
|
||||
original := &types.Message{
|
||||
Content: "",
|
||||
KnowledgeReferences: types.References{{
|
||||
Content: "chunk ",
|
||||
}},
|
||||
}
|
||||
messages := []*types.Message{original}
|
||||
|
||||
out := w.RewriteMessagesResponse(context.Background(), messages)
|
||||
|
||||
require.Len(t, out, 1)
|
||||
assert.NotSame(t, original, out[0])
|
||||
assert.Equal(t, "", original.Content)
|
||||
assert.Equal(t, "", out[0].Content)
|
||||
assert.Equal(t, "chunk ", original.KnowledgeReferences[0].Content)
|
||||
assert.Equal(t, "chunk ", out[0].KnowledgeReferences[0].Content)
|
||||
}
|
||||
|
||||
// SSE references payloads share their *SearchResult pointers with the stream
|
||||
// replay buffer and the assistant message being persisted, so rewriting must not
|
||||
// mutate the originals.
|
||||
|
|
|
|||
|
|
@ -62,10 +62,9 @@ type Resolver interface {
|
|||
// a signature.
|
||||
//
|
||||
// Logging policy:
|
||||
// - A successful rewrite logs at INFO with the full signed URL so operators
|
||||
// can copy it out of logs and verify public reachability directly. The
|
||||
// trade-off: anyone with log access can use a signed URL until it expires
|
||||
// (WeKnora grants 2h, MinIO 24h). Acceptable for diagnosability.
|
||||
// - A successful rewrite logs the source reference at INFO. The signed URL
|
||||
// is DEBUG-only so log aggregation cannot hand out anonymously readable
|
||||
// links; operators can raise the log level when verifying reachability.
|
||||
// - Failure or no-op logs at WARN. The no-op case usually means
|
||||
// APP_EXTERNAL_URL is unset, the most common cause of "image broken in the
|
||||
// IM channel / in my app" reports.
|
||||
|
|
@ -150,7 +149,8 @@ func (w *Rewriter) resolve(ctx context.Context, ref string) string {
|
|||
w.logPrefix, httpURL, ref)
|
||||
return ref
|
||||
}
|
||||
logger.Infof(ctx, "[%s] storage URL rewrite: src=%s dst=%s", w.logPrefix, ref, httpURL)
|
||||
logger.Infof(ctx, "[%s] storage URL rewrite: src=%s", w.logPrefix, ref)
|
||||
logger.Debugf(ctx, "[%s] storage URL rewrite dst=%s", w.logPrefix, httpURL)
|
||||
return httpURL
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue