fix(storage): let object storage transfers outlive 30 seconds (#3368)
* fix(storage): let object storage transfers outlive 30 seconds The S3, COS, KS3, OBS and OSS file services build their SDK client from DefaultSSRFSafeHTTPClientConfig, whose 30-second Timeout is Go's whole-request timeout, body included. So every request had to finish within 30 seconds whatever it carried: a 40 MB PDF needs about 11 Mbit/s of uplink to make it as one COS Object.Put (OSS uploads in 10 MB parts, so there the cap was per part), and the same file coming back from a distant region was cut the same way, while the local backend took any size (#3306). MinIO and TOS, which only hand the SDK a transport, never had the cap. Give the object storage clients a config without the whole-request timeout, and bound connection setup on their transport instead (TLS handshake, response headers, idle connections), so a peer that accepts the connection and then goes silent still fails within about a minute; the header wait starts after the request body is written, so it does not cap an upload. A transfer is then bounded by its request context. The bucket probe every constructor runs, which had relied on the client timeout, gets a 30-second context of its own, and the OSS and KS3 connectivity checks get the 10-second deadline the other backends already set. KS3 called its SDK without a context at all; it now passes the caller's, so the same bounds apply there. * fix(storage): bound object transfers and split bucket probes Dropping the 30s client timeout left asynq/API transfers unbounded, and Head+Create shared one setup context so a slow miss aborted create. Add a 30-minute fallback deadline, give each probe its own timeout, and only create a KS3 bucket on a real 404. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
014f95f139
commit
a5bdf82a02
|
|
@ -33,17 +33,15 @@ type cosFileService struct {
|
|||
const cosScheme = "cos://"
|
||||
|
||||
func newCOSHTTPClient(secretID, secretKey string) *http.Client {
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client := utils.NewSSRFSafeHTTPClient(httpConfig)
|
||||
client := objectStorageHTTPClient()
|
||||
// COS GetCredential expects AuthorizationTransport to remain the outermost
|
||||
// transport. Put the per-request SSRF guard immediately underneath it while
|
||||
// retaining the safe client's redirect policy.
|
||||
// transport. The per-request SSRF guard the client came with, over the
|
||||
// bounded transport, goes immediately underneath it; the client keeps its
|
||||
// redirect policy.
|
||||
client.Transport = &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
},
|
||||
Transport: client.Transport,
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
|
@ -120,6 +118,8 @@ func (s *cosFileService) SaveFile(ctx context.Context,
|
|||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.Object.Put(ctx, objectName, src, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to COS: %w", err)
|
||||
|
|
@ -136,11 +136,13 @@ func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.Re
|
|||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
resp, err := s.client.Object.Get(ctx, objectName, nil)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to get file from COS: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
return objectStorageBoundReader(resp.Body, cancel), nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from COS storage
|
||||
|
|
@ -152,6 +154,8 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
|
|||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.Object.Delete(ctx, objectName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
|
|
@ -209,6 +213,9 @@ func (s *cosFileService) CopyFile(ctx context.Context,
|
|||
destKey := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
// sourceURL is the host + object key WITHOUT a scheme, per the COS SDK contract.
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
sourceURL := fmt.Sprintf("%s.cos.%s.myqcloud.com/%s", s.bucketName, s.region, srcObjectKey)
|
||||
_, _, err = s.client.Object.Copy(ctx, destKey, sourceURL, nil)
|
||||
if err != nil {
|
||||
|
|
@ -229,9 +236,11 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
// 如果请求写入临时桶且临时桶已配置
|
||||
// ????????????????
|
||||
if temp && s.tempClient != nil {
|
||||
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
|
||||
|
|
@ -242,7 +251,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// 写入主桶
|
||||
// ????
|
||||
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
|
||||
_, err = s.client.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
|
|
@ -254,7 +263,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// 判断文件属于哪个桶
|
||||
// ?????????
|
||||
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
|
||||
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ package file
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -15,6 +17,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
ks3aws "github.com/ks3sdklib/aws-sdk-go/aws"
|
||||
"github.com/ks3sdklib/aws-sdk-go/aws/awserr"
|
||||
"github.com/ks3sdklib/aws-sdk-go/aws/credentials"
|
||||
ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
|
@ -65,20 +68,26 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err
|
|||
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
|
||||
SignerVersion: "V2", // KS3 recommends V2 signing
|
||||
MaxRetries: 3,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
HTTPClient: objectStorageHTTPClient(),
|
||||
})
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
|
||||
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
headCtx, cancel := objectStorageSetupContext()
|
||||
_, err := client.HeadBucketWithContext(headCtx, &ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// Bucket doesn't exist, try to create it
|
||||
_, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{
|
||||
if !isKS3BucketMissing(err) {
|
||||
return fmt.Errorf("failed to check KS3 bucket %q: %w", bucketName, err)
|
||||
}
|
||||
createCtx, cancel := objectStorageSetupContext()
|
||||
defer cancel()
|
||||
_, createErr := client.CreateBucketWithContext(createCtx, &ks3s3.CreateBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
if createErr != nil {
|
||||
|
|
@ -87,6 +96,30 @@ func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func isKS3BucketMissing(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var rf awserr.RequestFailure
|
||||
if errors.As(err, &rf) {
|
||||
if rf.StatusCode() == http.StatusNotFound {
|
||||
return true
|
||||
}
|
||||
switch rf.Code() {
|
||||
case "NoSuchBucket", "NotFound":
|
||||
return true
|
||||
}
|
||||
}
|
||||
var ae awserr.Error
|
||||
if errors.As(err, &ae) {
|
||||
switch ae.Code() {
|
||||
case "NoSuchBucket", "NotFound":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CheckKS3Connectivity tests KS3 connectivity using the provided credentials.
|
||||
func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
|
||||
|
|
@ -94,20 +127,12 @@ func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secr
|
|||
return err
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
return err
|
||||
}
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func joinKS3Key(parts ...string) string {
|
||||
|
|
@ -161,7 +186,9 @@ func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeade
|
|||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
Body: src,
|
||||
|
|
@ -182,7 +209,9 @@ func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
ext := filepath.Ext(safeName)
|
||||
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext)
|
||||
|
||||
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
Body: bytes.NewReader(data),
|
||||
|
|
@ -215,7 +244,9 @@ func (s *ks3FileService) CopyFile(ctx context.Context,
|
|||
ext := filepath.Ext(srcPath)
|
||||
destKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
|
||||
|
||||
_, err = s.client.CopyObject(&ks3s3.CopyObjectInput{
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.CopyObjectWithContext(ctx, &ks3s3.CopyObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(destKey),
|
||||
SourceBucket: ks3aws.String(s.bucketName),
|
||||
|
|
@ -239,15 +270,17 @@ func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadC
|
|||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
resp, err := s.client.GetObject(&ks3s3.GetObjectInput{
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
resp, err := s.client.GetObjectWithContext(ctx, &ks3s3.GetObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to get file from KS3: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
return objectStorageBoundReader(resp.Body, cancel), nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
|
|
@ -259,7 +292,9 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
|
|||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.DeleteObjectWithContext(ctx, &ks3s3.DeleteObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
})
|
||||
|
|
@ -270,19 +305,12 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
|
|||
}
|
||||
|
||||
func (s *ks3FileService) CheckConnectivity(ctx context.Context) error {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
return err
|
||||
}
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
// objectStorageSetupTimeout bounds the bucket probe a file service runs
|
||||
// when it is built. Services are rebuilt per resolution, so the probe
|
||||
// runs before many transfers, not just at startup.
|
||||
objectStorageSetupTimeout = 30 * time.Second
|
||||
|
||||
// objectStorageTransferTimeout is the fallback when the caller did not
|
||||
// set a deadline (asynq workers and the main HTTP server both omit one).
|
||||
// Long enough for a slow 40 MB COS PUT; short enough that a stalled body
|
||||
// cannot pin a worker forever.
|
||||
objectStorageTransferTimeout = 30 * time.Minute
|
||||
|
||||
// objectStorageResponseHeaderTimeout is longer than a typical GET/PUT
|
||||
// header wait because CopyObject / Object.Copy are synchronous: the
|
||||
// server does not send headers until the copy finishes. GET headers
|
||||
// still arrive immediately, so a silent peer fails in this window
|
||||
// rather than the full transfer budget.
|
||||
objectStorageResponseHeaderTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
// objectStorageHTTPClientConfig is the SSRF-safe client config for the object
|
||||
// storage SDKs. The default config's Timeout is Go's whole-request timeout,
|
||||
// body included, which capped every transfer at what a link moves in
|
||||
// 30 seconds (#3306). The object storage clients bound connection setup on
|
||||
// their transport instead and leave the transfer to the request context.
|
||||
func objectStorageHTTPClientConfig() utils.SSRFSafeHTTPClientConfig {
|
||||
config := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
config.Timeout = 0
|
||||
return config
|
||||
}
|
||||
|
||||
// objectStorageTransport is the SSRF-safe transport with the setup bounds the
|
||||
// whole-request timeout used to provide: a peer that accepts the connection
|
||||
// and then goes silent fails within the header wait instead of holding the
|
||||
// request until its context ends. The header wait starts after the request
|
||||
// body is written, so it does not cap an upload.
|
||||
func objectStorageTransport() *http.Transport {
|
||||
transport := utils.NewSSRFSafeTransport(objectStorageHTTPClientConfig())
|
||||
transport.TLSHandshakeTimeout = 10 * time.Second
|
||||
transport.ResponseHeaderTimeout = objectStorageResponseHeaderTimeout
|
||||
transport.ExpectContinueTimeout = time.Second
|
||||
transport.IdleConnTimeout = 90 * time.Second
|
||||
return transport
|
||||
}
|
||||
|
||||
// objectStorageHTTPClient is the client for the SDKs that take a whole
|
||||
// http.Client (S3, KS3, OBS, OSS). COS wraps objectStorageTransport in its
|
||||
// own signing transport.
|
||||
func objectStorageHTTPClient() *http.Client {
|
||||
return utils.NewSSRFSafeHTTPClientWithTransport(objectStorageHTTPClientConfig(), objectStorageTransport())
|
||||
}
|
||||
|
||||
// objectStorageSetupContext bounds one constructor probe (HeadBucket or
|
||||
// CreateBucket). Callers must not reuse the same context for both: a slow
|
||||
// Head that then reports "missing" would leave Create with no time left.
|
||||
func objectStorageSetupContext() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), objectStorageSetupTimeout)
|
||||
}
|
||||
|
||||
// objectStorageTransferContext adds a fallback deadline only when the caller
|
||||
// did not set one, matching models/chat.withLLMTimeout.
|
||||
func objectStorageTransferContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return ctx, func() {}
|
||||
}
|
||||
return context.WithTimeout(ctx, objectStorageTransferTimeout)
|
||||
}
|
||||
|
||||
type objectStorageBoundReadCloser struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func objectStorageBoundReader(body io.ReadCloser, cancel context.CancelFunc) io.ReadCloser {
|
||||
if body == nil {
|
||||
cancel()
|
||||
return nil
|
||||
}
|
||||
return &objectStorageBoundReadCloser{ReadCloser: body, cancel: cancel}
|
||||
}
|
||||
|
||||
func (r *objectStorageBoundReadCloser) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
r.cancel()
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ks3sdklib/aws-sdk-go/aws/awserr"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// The object storage clients must not carry Go's whole-request timeout: it
|
||||
// bounds the body transfer too, so it capped every upload and download at
|
||||
// what a link can move in 30 seconds (#3306).
|
||||
func TestObjectStorageHTTPClientConfigDropsOnlyTheWholeRequestTimeout(t *testing.T) {
|
||||
want := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
want.Timeout = 0
|
||||
|
||||
require.Equal(t, want, objectStorageHTTPClientConfig())
|
||||
}
|
||||
|
||||
// Without the whole-request timeout, connection setup is bounded on the
|
||||
// transport instead; a silent peer must still fail in bounded time.
|
||||
func TestObjectStorageTransportBoundsConnectionSetup(t *testing.T) {
|
||||
transport := objectStorageTransport()
|
||||
|
||||
require.Equal(t, 10*time.Second, transport.TLSHandshakeTimeout)
|
||||
require.Equal(t, objectStorageResponseHeaderTimeout, transport.ResponseHeaderTimeout)
|
||||
require.Equal(t, time.Second, transport.ExpectContinueTimeout)
|
||||
require.Equal(t, 90*time.Second, transport.IdleConnTimeout)
|
||||
require.NotNil(t, transport.DialContext, "the SSRF-safe dialer must stay in place")
|
||||
}
|
||||
|
||||
func TestObjectStorageHTTPClientKeepsTheSSRFGuard(t *testing.T) {
|
||||
client := objectStorageHTTPClient()
|
||||
|
||||
require.Zero(t, client.Timeout)
|
||||
_, ok := client.Transport.(*utils.SSRFValidatingRoundTripper)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, client.CheckRedirect)
|
||||
}
|
||||
|
||||
func TestCOSHTTPClientHasNoWholeRequestTimeout(t *testing.T) {
|
||||
client := newCOSHTTPClient("id", "key")
|
||||
|
||||
require.Zero(t, client.Timeout)
|
||||
auth, ok := client.Transport.(*cos.AuthorizationTransport)
|
||||
require.True(t, ok, "COS signing must stay the outermost transport")
|
||||
guard, ok := auth.Transport.(*utils.SSRFValidatingRoundTripper)
|
||||
require.True(t, ok, "the SSRF guard must stay underneath the signer")
|
||||
base, ok := guard.Base.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, objectStorageResponseHeaderTimeout, base.ResponseHeaderTimeout)
|
||||
}
|
||||
|
||||
func TestOBSClientHasNoWholeRequestTimeout(t *testing.T) {
|
||||
options := obsS3Options("obs.cn-north-4.myhuaweicloud.com", "cn-north-4", "ak", "sk")
|
||||
|
||||
client, ok := options.HTTPClient.(*http.Client)
|
||||
require.True(t, ok)
|
||||
require.Zero(t, client.Timeout)
|
||||
}
|
||||
|
||||
func TestS3ClientHasNoWholeRequestTimeout(t *testing.T) {
|
||||
svc, err := newS3Client("", "ak", "sk", "bucket", "us-east-1", "", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
client, ok := svc.client.Options().HTTPClient.(*http.Client)
|
||||
require.True(t, ok)
|
||||
require.Zero(t, client.Timeout)
|
||||
}
|
||||
|
||||
func TestKS3ClientHasNoWholeRequestTimeout(t *testing.T) {
|
||||
// The endpoint check resolves the host unless it is whitelisted; keep the
|
||||
// test off the network.
|
||||
t.Setenv("SSRF_WHITELIST", "ks3-cn-beijing.ksyuncs.com")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(utils.ResetSSRFWhitelistForTest)
|
||||
client, err := newKS3Client("https://ks3-cn-beijing.ksyuncs.com", "cn-beijing", "ak", "sk")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Zero(t, client.Config.HTTPClient.Timeout)
|
||||
}
|
||||
|
||||
func TestOSSClientHasNoWholeRequestTimeout(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "oss-cn-hangzhou.aliyuncs.com")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(utils.ResetSSRFWhitelistForTest)
|
||||
client, err := newOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "cn-hangzhou", "ak", "sk")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, client)
|
||||
require.Zero(t, objectStorageHTTPClient().Timeout)
|
||||
}
|
||||
|
||||
func TestObjectStorageTransferContextAddsFallbackDeadline(t *testing.T) {
|
||||
ctx, cancel := objectStorageTransferContext(context.Background())
|
||||
defer cancel()
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
require.True(t, ok)
|
||||
remain := time.Until(deadline)
|
||||
require.Greater(t, remain, 29*time.Minute)
|
||||
require.Less(t, remain, 31*time.Minute)
|
||||
}
|
||||
|
||||
func TestObjectStorageTransferContextKeepsCallerDeadline(t *testing.T) {
|
||||
parent, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ctx, stop := objectStorageTransferContext(parent)
|
||||
defer stop()
|
||||
require.Equal(t, parent, ctx)
|
||||
}
|
||||
|
||||
func TestObjectStorageBoundReaderCancelsOnClose(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := objectStorageBoundReader(io.NopCloser(strings.NewReader("x")), cancel)
|
||||
require.NoError(t, body.Close())
|
||||
require.ErrorIs(t, ctx.Err(), context.Canceled)
|
||||
}
|
||||
|
||||
func TestKS3BucketMissingDetection(t *testing.T) {
|
||||
require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
|
||||
awserr.New("NoSuchBucket", "missing", nil), http.StatusNotFound, "req")))
|
||||
require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
|
||||
awserr.New("NotFound", "missing", nil), http.StatusNotFound, "req")))
|
||||
require.False(t, isKS3BucketMissing(awserr.NewRequestFailure(
|
||||
awserr.New("AccessDenied", "denied", nil), http.StatusForbidden, "req")))
|
||||
require.False(t, isKS3BucketMissing(context.DeadlineExceeded))
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package file
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
|
|
@ -18,6 +19,7 @@ import (
|
|||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
|
@ -54,7 +56,7 @@ func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Option
|
|||
// trailing checksum negotiation; align with the S3 driver (s3.go).
|
||||
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
|
||||
UsePathStyle: obsUsePathStyle(endpoint),
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
HTTPClient: objectStorageHTTPClient(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -68,15 +70,24 @@ func NewObsFileService(
|
|||
|
||||
client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey))
|
||||
|
||||
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
|
||||
headCtx, cancel := objectStorageSetupContext()
|
||||
_, err := client.HeadBucket(headCtx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
_, createErr := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if createErr != nil {
|
||||
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
|
||||
var notFound *s3types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
createCtx, createCancel := objectStorageSetupContext()
|
||||
_, createErr := client.CreateBucket(createCtx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
createCancel()
|
||||
if createErr != nil {
|
||||
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -176,6 +187,8 @@ func (s *obsFileService) SaveFile(ctx context.Context,
|
|||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
|
|
@ -200,15 +213,17 @@ func (s *obsFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
|
|||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
output, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to get file from OBS: %w", err)
|
||||
}
|
||||
|
||||
return output.Body, nil
|
||||
return objectStorageBoundReader(output.Body, cancel), nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
|
|
@ -217,6 +232,8 @@ func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error
|
|||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
|
|
@ -285,6 +302,8 @@ func (s *obsFileService) CopyFile(ctx context.Context,
|
|||
|
||||
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
|
||||
// (url.PathEscape would turn them into %2F and break the bucket/key split).
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
CopySource: aws.String(s.bucketName + "/" + srcKey),
|
||||
|
|
@ -323,6 +342,8 @@ func (s *obsFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
}
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
|
|
|
|||
|
|
@ -43,14 +43,16 @@ func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, e
|
|||
WithCredentialsProvider(creds).
|
||||
WithRegion(region).
|
||||
WithEndpoint(endpoint).
|
||||
WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()))
|
||||
WithHttpClient(objectStorageHTTPClient())
|
||||
|
||||
return oss.NewClient(cfg), nil
|
||||
}
|
||||
|
||||
// ossEnsureBucket checks if the bucket exists and creates it if missing.
|
||||
func ossEnsureBucket(client *oss.Client, bucketName string) error {
|
||||
exists, err := client.IsBucketExist(context.Background(), bucketName)
|
||||
headCtx, cancel := objectStorageSetupContext()
|
||||
exists, err := client.IsBucketExist(headCtx, bucketName)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check OSS bucket: %w", err)
|
||||
}
|
||||
|
|
@ -58,7 +60,9 @@ func ossEnsureBucket(client *oss.Client, bucketName string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
_, err = client.PutBucket(context.Background(), &oss.PutBucketRequest{
|
||||
createCtx, cancel := objectStorageSetupContext()
|
||||
defer cancel()
|
||||
_, err = client.PutBucket(createCtx, &oss.PutBucketRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -123,7 +127,9 @@ func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secr
|
|||
return err
|
||||
}
|
||||
|
||||
exists, err := client.IsBucketExist(ctx, bucketName)
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
exists, err := client.IsBucketExist(checkCtx, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check OSS bucket: %w", err)
|
||||
}
|
||||
|
|
@ -180,6 +186,9 @@ func (s *ossFileService) SaveFile(ctx context.Context,
|
|||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Use Uploader for files > 10MB (auto multipart with concurrent uploads)
|
||||
const multipartThreshold = 10 * 1024 * 1024
|
||||
if file.Size > multipartThreshold {
|
||||
|
|
@ -234,6 +243,8 @@ func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
|
|||
objectName = fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = client.PutObject(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(targetBucket),
|
||||
Key: oss.Ptr(objectName),
|
||||
|
|
@ -264,6 +275,8 @@ func (s *ossFileService) CopyFile(ctx context.Context,
|
|||
ext := filepath.Ext(srcPath)
|
||||
destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = s.client.CopyObject(ctx, &oss.CopyObjectRequest{
|
||||
Bucket: oss.Ptr(s.bucketName),
|
||||
Key: oss.Ptr(destKey),
|
||||
|
|
@ -296,15 +309,17 @@ func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
|
|||
client = s.client
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
resp, err := client.GetObject(ctx, &oss.GetObjectRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
return objectStorageBoundReader(resp.Body, cancel), nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from OSS.
|
||||
|
|
@ -324,6 +339,8 @@ func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error
|
|||
client = s.client
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
_, err = client.DeleteObject(ctx, &oss.DeleteObjectRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
|
|||
// Create S3 client with custom endpoint if provided.
|
||||
// For S3-compatible services (non-AWS), use path-style addressing
|
||||
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
|
||||
httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())
|
||||
httpClient := objectStorageHTTPClient()
|
||||
var client *s3.Client
|
||||
if endpoint != "" {
|
||||
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
|
||||
|
|
@ -101,19 +101,9 @@ func NewS3FileService(endpoint,
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if bucket exists
|
||||
exists, err := svc.bucketExists(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
if err := ensureS3Bucket(svc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if err = svc.createBucket(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
|
|
@ -124,18 +114,30 @@ func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, reg
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exists, err := svc.bucketExists(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err = svc.createBucket(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
if err := ensureS3Bucket(svc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
func ensureS3Bucket(svc *s3FileService) error {
|
||||
headCtx, cancel := objectStorageSetupContext()
|
||||
exists, err := svc.bucketExists(headCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
createCtx, cancel := objectStorageSetupContext()
|
||||
defer cancel()
|
||||
if err := svc.createBucket(createCtx); err != nil {
|
||||
return fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bucketExists checks if the bucket exists
|
||||
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
|
||||
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
|
|
@ -238,6 +240,9 @@ func (s *s3FileService) SaveFile(ctx context.Context,
|
|||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Upload file to S3
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
|
|
@ -260,15 +265,17 @@ func (s *s3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCl
|
|||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to get file from S3: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
return objectStorageBoundReader(resp.Body, cancel), nil
|
||||
}
|
||||
|
||||
// DeleteFile deletes a file
|
||||
|
|
@ -278,6 +285,9 @@ func (s *s3FileService) DeleteFile(ctx context.Context, filePath string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
|
|
@ -306,6 +316,9 @@ func (s *s3FileService) CopyFile(ctx context.Context,
|
|||
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
|
||||
// (url.PathEscape would turn them into %2F and break the bucket/key split).
|
||||
// srcKey is already validated by parseS3FilePath -> SafeObjectKey.
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
CopySource: aws.String(s.bucketName + "/" + srcKey),
|
||||
|
|
@ -330,6 +343,9 @@ func (s *s3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uin
|
|||
ext := filepath.Ext(safeName)
|
||||
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
|
||||
ctx, cancel := objectStorageTransferContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Upload bytes to S3
|
||||
reader := bytes.NewReader(data)
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
|
|
|
|||
|
|
@ -582,7 +582,11 @@ func registerChatLocalImageResolver(
|
|||
resourceCatalog interfaces.ResourceCatalog,
|
||||
) {
|
||||
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
|
||||
ctx := context.Background()
|
||||
// The object storage clients bound connection setup but leave the
|
||||
// transfer to this context, so give it a deadline: a chat turn must
|
||||
// not hang on one image whose download stalls.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
|
|
|
|||
Loading…
Reference in New Issue