fix(storage): let object storage transfers outlive 30 seconds (#3368)

* fix(storage): let object storage transfers outlive 30 seconds

The S3, COS, KS3, OBS and OSS file services build their SDK client from
DefaultSSRFSafeHTTPClientConfig, whose 30-second Timeout is Go's
whole-request timeout, body included. So every request had to finish
within 30 seconds whatever it carried: a 40 MB PDF needs about
11 Mbit/s of uplink to make it as one COS Object.Put (OSS uploads in
10 MB parts, so there the cap was per part), and the same file coming
back from a distant region was cut the same way, while the local backend
took any size (#3306). MinIO and TOS, which only hand the SDK a
transport, never had the cap.

Give the object storage clients a config without the whole-request
timeout, and bound connection setup on their transport instead (TLS
handshake, response headers, idle connections), so a peer that accepts
the connection and then goes silent still fails within about a minute;
the header wait starts after the request body is written, so it does
not cap an upload. A transfer is then bounded by its request context.
The bucket probe every constructor runs, which had relied on the client
timeout, gets a 30-second context of its own, and the OSS and KS3
connectivity checks get the 10-second deadline the other backends
already set. KS3 called its SDK without a context at all; it now passes
the caller's, so the same bounds apply there.

* fix(storage): bound object transfers and split bucket probes

Dropping the 30s client timeout left asynq/API transfers unbounded, and Head+Create shared one setup context so a slow miss aborted create. Add a 30-minute fallback deadline, give each probe its own timeout, and only create a KS3 bucket on a real 404.

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
main
Captain 2026-09-19 21:30:55 +08:00 committed by GitHub
parent 014f95f139
commit a5bdf82a02
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 415 additions and 84 deletions

View File

@ -33,17 +33,15 @@ type cosFileService struct {
const cosScheme = "cos://"
func newCOSHTTPClient(secretID, secretKey string) *http.Client {
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
client := utils.NewSSRFSafeHTTPClient(httpConfig)
client := objectStorageHTTPClient()
// COS GetCredential expects AuthorizationTransport to remain the outermost
// transport. Put the per-request SSRF guard immediately underneath it while
// retaining the safe client's redirect policy.
// transport. The per-request SSRF guard the client came with, over the
// bounded transport, goes immediately underneath it; the client keeps its
// redirect policy.
client.Transport = &cos.AuthorizationTransport{
SecretID: secretID,
SecretKey: secretKey,
Transport: &utils.SSRFValidatingRoundTripper{
Base: utils.NewSSRFSafeTransport(httpConfig),
},
Transport: client.Transport,
}
return client
}
@ -120,6 +118,8 @@ func (s *cosFileService) SaveFile(ctx context.Context,
return "", fmt.Errorf("failed to open file: %w", err)
}
defer src.Close()
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.Object.Put(ctx, objectName, src, nil)
if err != nil {
return "", fmt.Errorf("failed to upload file to COS: %w", err)
@ -136,11 +136,13 @@ func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.Re
if err := utils.SafeObjectKey(objectName); err != nil {
return nil, fmt.Errorf("invalid file path: %w", err)
}
ctx, cancel := objectStorageTransferContext(ctx)
resp, err := s.client.Object.Get(ctx, objectName, nil)
if err != nil {
cancel()
return nil, fmt.Errorf("failed to get file from COS: %w", err)
}
return resp.Body, nil
return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile removes a file from COS storage
@ -152,6 +154,8 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
if err := utils.SafeObjectKey(objectName); err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.Object.Delete(ctx, objectName)
if err != nil {
return fmt.Errorf("failed to delete file: %w", err)
@ -209,6 +213,9 @@ func (s *cosFileService) CopyFile(ctx context.Context,
destKey := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
// sourceURL is the host + object key WITHOUT a scheme, per the COS SDK contract.
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
sourceURL := fmt.Sprintf("%s.cos.%s.myqcloud.com/%s", s.bucketName, s.region, srcObjectKey)
_, _, err = s.client.Object.Copy(ctx, destKey, sourceURL, nil)
if err != nil {
@ -229,9 +236,11 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
return "", fmt.Errorf("invalid file name: %w", err)
}
ext := filepath.Ext(safeName)
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
reader := bytes.NewReader(data)
// 如果请求写入临时桶且临时桶已配置
// ????????????????
if temp && s.tempClient != nil {
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
@ -242,7 +251,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
}
// 写入主桶
// ????
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
_, err = s.client.Object.Put(ctx, objectName, reader, nil)
if err != nil {
@ -254,7 +263,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
// GetFileURL returns a presigned download URL for the file
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// 判断文件属于哪个桶
// ?????????
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
if err := utils.SafeObjectKey(objectName); err != nil {

View File

@ -3,9 +3,11 @@ package file
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"path/filepath"
"strings"
"time"
@ -15,6 +17,7 @@ import (
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
ks3aws "github.com/ks3sdklib/aws-sdk-go/aws"
"github.com/ks3sdklib/aws-sdk-go/aws/awserr"
"github.com/ks3sdklib/aws-sdk-go/aws/credentials"
ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3"
)
@ -65,20 +68,26 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
SignerVersion: "V2", // KS3 recommends V2 signing
MaxRetries: 3,
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
HTTPClient: objectStorageHTTPClient(),
})
return client, nil
}
func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
headCtx, cancel := objectStorageSetupContext()
_, err := client.HeadBucketWithContext(headCtx, &ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
cancel()
if err == nil {
return nil
}
// Bucket doesn't exist, try to create it
_, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{
if !isKS3BucketMissing(err) {
return fmt.Errorf("failed to check KS3 bucket %q: %w", bucketName, err)
}
createCtx, cancel := objectStorageSetupContext()
defer cancel()
_, createErr := client.CreateBucketWithContext(createCtx, &ks3s3.CreateBucketInput{
Bucket: ks3aws.String(bucketName),
})
if createErr != nil {
@ -87,6 +96,30 @@ func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
return nil
}
func isKS3BucketMissing(err error) bool {
if err == nil {
return false
}
var rf awserr.RequestFailure
if errors.As(err, &rf) {
if rf.StatusCode() == http.StatusNotFound {
return true
}
switch rf.Code() {
case "NoSuchBucket", "NotFound":
return true
}
}
var ae awserr.Error
if errors.As(err, &ae) {
switch ae.Code() {
case "NoSuchBucket", "NotFound":
return true
}
}
return false
}
// CheckKS3Connectivity tests KS3 connectivity using the provided credentials.
func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
@ -94,20 +127,12 @@ func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secr
return err
}
done := make(chan error, 1)
go func() {
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
done <- err
}()
select {
case <-ctx.Done():
return ctx.Err()
case err := <-done:
return err
}
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err = client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
return err
}
func joinKS3Key(parts ...string) string {
@ -161,7 +186,9 @@ func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeade
contentType = utils.GetContentTypeByExt(ext)
}
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: src,
@ -182,7 +209,9 @@ func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
ext := filepath.Ext(safeName)
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext)
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: bytes.NewReader(data),
@ -215,7 +244,9 @@ func (s *ks3FileService) CopyFile(ctx context.Context,
ext := filepath.Ext(srcPath)
destKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
_, err = s.client.CopyObject(&ks3s3.CopyObjectInput{
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.CopyObjectWithContext(ctx, &ks3s3.CopyObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(destKey),
SourceBucket: ks3aws.String(s.bucketName),
@ -239,15 +270,17 @@ func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadC
return nil, fmt.Errorf("invalid file path: %w", err)
}
resp, err := s.client.GetObject(&ks3s3.GetObjectInput{
ctx, cancel := objectStorageTransferContext(ctx)
resp, err := s.client.GetObjectWithContext(ctx, &ks3s3.GetObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
if err != nil {
cancel()
return nil, fmt.Errorf("failed to get file from KS3: %w", err)
}
return resp.Body, nil
return objectStorageBoundReader(resp.Body, cancel), nil
}
func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error {
@ -259,7 +292,9 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
return fmt.Errorf("invalid file path: %w", err)
}
_, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.DeleteObjectWithContext(ctx, &ks3s3.DeleteObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
@ -270,19 +305,12 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
}
func (s *ks3FileService) CheckConnectivity(ctx context.Context) error {
done := make(chan error, 1)
go func() {
_, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{
Bucket: ks3aws.String(s.bucketName),
})
done <- err
}()
select {
case <-ctx.Done():
return ctx.Err()
case err := <-done:
return err
}
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := s.client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
Bucket: ks3aws.String(s.bucketName),
})
return err
}
func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {

View File

@ -0,0 +1,100 @@
package file
import (
"context"
"io"
"net/http"
"time"
"github.com/Tencent/WeKnora/internal/utils"
)
const (
// objectStorageSetupTimeout bounds the bucket probe a file service runs
// when it is built. Services are rebuilt per resolution, so the probe
// runs before many transfers, not just at startup.
objectStorageSetupTimeout = 30 * time.Second
// objectStorageTransferTimeout is the fallback when the caller did not
// set a deadline (asynq workers and the main HTTP server both omit one).
// Long enough for a slow 40 MB COS PUT; short enough that a stalled body
// cannot pin a worker forever.
objectStorageTransferTimeout = 30 * time.Minute
// objectStorageResponseHeaderTimeout is longer than a typical GET/PUT
// header wait because CopyObject / Object.Copy are synchronous: the
// server does not send headers until the copy finishes. GET headers
// still arrive immediately, so a silent peer fails in this window
// rather than the full transfer budget.
objectStorageResponseHeaderTimeout = 10 * time.Minute
)
// objectStorageHTTPClientConfig is the SSRF-safe client config for the object
// storage SDKs. The default config's Timeout is Go's whole-request timeout,
// body included, which capped every transfer at what a link moves in
// 30 seconds (#3306). The object storage clients bound connection setup on
// their transport instead and leave the transfer to the request context.
func objectStorageHTTPClientConfig() utils.SSRFSafeHTTPClientConfig {
config := utils.DefaultSSRFSafeHTTPClientConfig()
config.Timeout = 0
return config
}
// objectStorageTransport is the SSRF-safe transport with the setup bounds the
// whole-request timeout used to provide: a peer that accepts the connection
// and then goes silent fails within the header wait instead of holding the
// request until its context ends. The header wait starts after the request
// body is written, so it does not cap an upload.
func objectStorageTransport() *http.Transport {
transport := utils.NewSSRFSafeTransport(objectStorageHTTPClientConfig())
transport.TLSHandshakeTimeout = 10 * time.Second
transport.ResponseHeaderTimeout = objectStorageResponseHeaderTimeout
transport.ExpectContinueTimeout = time.Second
transport.IdleConnTimeout = 90 * time.Second
return transport
}
// objectStorageHTTPClient is the client for the SDKs that take a whole
// http.Client (S3, KS3, OBS, OSS). COS wraps objectStorageTransport in its
// own signing transport.
func objectStorageHTTPClient() *http.Client {
return utils.NewSSRFSafeHTTPClientWithTransport(objectStorageHTTPClientConfig(), objectStorageTransport())
}
// objectStorageSetupContext bounds one constructor probe (HeadBucket or
// CreateBucket). Callers must not reuse the same context for both: a slow
// Head that then reports "missing" would leave Create with no time left.
func objectStorageSetupContext() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), objectStorageSetupTimeout)
}
// objectStorageTransferContext adds a fallback deadline only when the caller
// did not set one, matching models/chat.withLLMTimeout.
func objectStorageTransferContext(ctx context.Context) (context.Context, context.CancelFunc) {
if ctx == nil {
ctx = context.Background()
}
if _, ok := ctx.Deadline(); ok {
return ctx, func() {}
}
return context.WithTimeout(ctx, objectStorageTransferTimeout)
}
type objectStorageBoundReadCloser struct {
io.ReadCloser
cancel context.CancelFunc
}
func objectStorageBoundReader(body io.ReadCloser, cancel context.CancelFunc) io.ReadCloser {
if body == nil {
cancel()
return nil
}
return &objectStorageBoundReadCloser{ReadCloser: body, cancel: cancel}
}
func (r *objectStorageBoundReadCloser) Close() error {
err := r.ReadCloser.Close()
r.cancel()
return err
}

View File

@ -0,0 +1,136 @@
package file
import (
"context"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/ks3sdklib/aws-sdk-go/aws/awserr"
"github.com/stretchr/testify/require"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/Tencent/WeKnora/internal/utils"
)
// The object storage clients must not carry Go's whole-request timeout: it
// bounds the body transfer too, so it capped every upload and download at
// what a link can move in 30 seconds (#3306).
func TestObjectStorageHTTPClientConfigDropsOnlyTheWholeRequestTimeout(t *testing.T) {
want := utils.DefaultSSRFSafeHTTPClientConfig()
want.Timeout = 0
require.Equal(t, want, objectStorageHTTPClientConfig())
}
// Without the whole-request timeout, connection setup is bounded on the
// transport instead; a silent peer must still fail in bounded time.
func TestObjectStorageTransportBoundsConnectionSetup(t *testing.T) {
transport := objectStorageTransport()
require.Equal(t, 10*time.Second, transport.TLSHandshakeTimeout)
require.Equal(t, objectStorageResponseHeaderTimeout, transport.ResponseHeaderTimeout)
require.Equal(t, time.Second, transport.ExpectContinueTimeout)
require.Equal(t, 90*time.Second, transport.IdleConnTimeout)
require.NotNil(t, transport.DialContext, "the SSRF-safe dialer must stay in place")
}
func TestObjectStorageHTTPClientKeepsTheSSRFGuard(t *testing.T) {
client := objectStorageHTTPClient()
require.Zero(t, client.Timeout)
_, ok := client.Transport.(*utils.SSRFValidatingRoundTripper)
require.True(t, ok)
require.NotNil(t, client.CheckRedirect)
}
func TestCOSHTTPClientHasNoWholeRequestTimeout(t *testing.T) {
client := newCOSHTTPClient("id", "key")
require.Zero(t, client.Timeout)
auth, ok := client.Transport.(*cos.AuthorizationTransport)
require.True(t, ok, "COS signing must stay the outermost transport")
guard, ok := auth.Transport.(*utils.SSRFValidatingRoundTripper)
require.True(t, ok, "the SSRF guard must stay underneath the signer")
base, ok := guard.Base.(*http.Transport)
require.True(t, ok)
require.Equal(t, objectStorageResponseHeaderTimeout, base.ResponseHeaderTimeout)
}
func TestOBSClientHasNoWholeRequestTimeout(t *testing.T) {
options := obsS3Options("obs.cn-north-4.myhuaweicloud.com", "cn-north-4", "ak", "sk")
client, ok := options.HTTPClient.(*http.Client)
require.True(t, ok)
require.Zero(t, client.Timeout)
}
func TestS3ClientHasNoWholeRequestTimeout(t *testing.T) {
svc, err := newS3Client("", "ak", "sk", "bucket", "us-east-1", "", false)
require.NoError(t, err)
client, ok := svc.client.Options().HTTPClient.(*http.Client)
require.True(t, ok)
require.Zero(t, client.Timeout)
}
func TestKS3ClientHasNoWholeRequestTimeout(t *testing.T) {
// The endpoint check resolves the host unless it is whitelisted; keep the
// test off the network.
t.Setenv("SSRF_WHITELIST", "ks3-cn-beijing.ksyuncs.com")
utils.ResetSSRFWhitelistForTest()
t.Cleanup(utils.ResetSSRFWhitelistForTest)
client, err := newKS3Client("https://ks3-cn-beijing.ksyuncs.com", "cn-beijing", "ak", "sk")
require.NoError(t, err)
require.Zero(t, client.Config.HTTPClient.Timeout)
}
func TestOSSClientHasNoWholeRequestTimeout(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "oss-cn-hangzhou.aliyuncs.com")
utils.ResetSSRFWhitelistForTest()
t.Cleanup(utils.ResetSSRFWhitelistForTest)
client, err := newOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "cn-hangzhou", "ak", "sk")
require.NoError(t, err)
require.NotNil(t, client)
require.Zero(t, objectStorageHTTPClient().Timeout)
}
func TestObjectStorageTransferContextAddsFallbackDeadline(t *testing.T) {
ctx, cancel := objectStorageTransferContext(context.Background())
defer cancel()
deadline, ok := ctx.Deadline()
require.True(t, ok)
remain := time.Until(deadline)
require.Greater(t, remain, 29*time.Minute)
require.Less(t, remain, 31*time.Minute)
}
func TestObjectStorageTransferContextKeepsCallerDeadline(t *testing.T) {
parent, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ctx, stop := objectStorageTransferContext(parent)
defer stop()
require.Equal(t, parent, ctx)
}
func TestObjectStorageBoundReaderCancelsOnClose(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
body := objectStorageBoundReader(io.NopCloser(strings.NewReader("x")), cancel)
require.NoError(t, body.Close())
require.ErrorIs(t, ctx.Err(), context.Canceled)
}
func TestKS3BucketMissingDetection(t *testing.T) {
require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
awserr.New("NoSuchBucket", "missing", nil), http.StatusNotFound, "req")))
require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
awserr.New("NotFound", "missing", nil), http.StatusNotFound, "req")))
require.False(t, isKS3BucketMissing(awserr.NewRequestFailure(
awserr.New("AccessDenied", "denied", nil), http.StatusForbidden, "req")))
require.False(t, isKS3BucketMissing(context.DeadlineExceeded))
}

View File

@ -2,6 +2,7 @@ package file
import (
"context"
"errors"
"fmt"
"io"
"mime/multipart"
@ -18,6 +19,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
)
@ -54,7 +56,7 @@ func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Option
// trailing checksum negotiation; align with the S3 driver (s3.go).
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
UsePathStyle: obsUsePathStyle(endpoint),
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
HTTPClient: objectStorageHTTPClient(),
}
}
@ -68,15 +70,24 @@ func NewObsFileService(
client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey))
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
headCtx, cancel := objectStorageSetupContext()
_, err := client.HeadBucket(headCtx, &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
})
cancel()
if err != nil {
_, createErr := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
if createErr != nil {
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
var notFound *s3types.NotFound
if errors.As(err, &notFound) {
createCtx, createCancel := objectStorageSetupContext()
_, createErr := client.CreateBucket(createCtx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
createCancel()
if createErr != nil {
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
}
} else {
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, err)
}
}
@ -176,6 +187,8 @@ func (s *obsFileService) SaveFile(ctx context.Context,
contentType = "application/octet-stream"
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
@ -200,15 +213,17 @@ func (s *obsFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
return nil, err
}
ctx, cancel := objectStorageTransferContext(ctx)
output, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
cancel()
return nil, fmt.Errorf("failed to get file from OBS: %w", err)
}
return output.Body, nil
return objectStorageBoundReader(output.Body, cancel), nil
}
func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error {
@ -217,6 +232,8 @@ func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error
return err
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
@ -285,6 +302,8 @@ func (s *obsFileService) CopyFile(ctx context.Context,
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
// (url.PathEscape would turn them into %2F and break the bucket/key split).
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(s.bucketName),
CopySource: aws.String(s.bucketName + "/" + srcKey),
@ -323,6 +342,8 @@ func (s *obsFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
}
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),

View File

@ -43,14 +43,16 @@ func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, e
WithCredentialsProvider(creds).
WithRegion(region).
WithEndpoint(endpoint).
WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()))
WithHttpClient(objectStorageHTTPClient())
return oss.NewClient(cfg), nil
}
// ossEnsureBucket checks if the bucket exists and creates it if missing.
func ossEnsureBucket(client *oss.Client, bucketName string) error {
exists, err := client.IsBucketExist(context.Background(), bucketName)
headCtx, cancel := objectStorageSetupContext()
exists, err := client.IsBucketExist(headCtx, bucketName)
cancel()
if err != nil {
return fmt.Errorf("failed to check OSS bucket: %w", err)
}
@ -58,7 +60,9 @@ func ossEnsureBucket(client *oss.Client, bucketName string) error {
return nil
}
_, err = client.PutBucket(context.Background(), &oss.PutBucketRequest{
createCtx, cancel := objectStorageSetupContext()
defer cancel()
_, err = client.PutBucket(createCtx, &oss.PutBucketRequest{
Bucket: oss.Ptr(bucketName),
})
if err != nil {
@ -123,7 +127,9 @@ func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secr
return err
}
exists, err := client.IsBucketExist(ctx, bucketName)
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
exists, err := client.IsBucketExist(checkCtx, bucketName)
if err != nil {
return fmt.Errorf("failed to check OSS bucket: %w", err)
}
@ -180,6 +186,9 @@ func (s *ossFileService) SaveFile(ctx context.Context,
contentType = utils.GetContentTypeByExt(ext)
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
// Use Uploader for files > 10MB (auto multipart with concurrent uploads)
const multipartThreshold = 10 * 1024 * 1024
if file.Size > multipartThreshold {
@ -234,6 +243,8 @@ func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
objectName = fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = client.PutObject(ctx, &oss.PutObjectRequest{
Bucket: oss.Ptr(targetBucket),
Key: oss.Ptr(objectName),
@ -264,6 +275,8 @@ func (s *ossFileService) CopyFile(ctx context.Context,
ext := filepath.Ext(srcPath)
destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.CopyObject(ctx, &oss.CopyObjectRequest{
Bucket: oss.Ptr(s.bucketName),
Key: oss.Ptr(destKey),
@ -296,15 +309,17 @@ func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
client = s.client
}
ctx, cancel := objectStorageTransferContext(ctx)
resp, err := client.GetObject(ctx, &oss.GetObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(objectName),
})
if err != nil {
cancel()
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
}
return resp.Body, nil
return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile removes a file from OSS.
@ -324,6 +339,8 @@ func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error
client = s.client
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = client.DeleteObject(ctx, &oss.DeleteObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(objectName),

View File

@ -58,7 +58,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
// Create S3 client with custom endpoint if provided.
// For S3-compatible services (non-AWS), use path-style addressing
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())
httpClient := objectStorageHTTPClient()
var client *s3.Client
if endpoint != "" {
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
@ -101,19 +101,9 @@ func NewS3FileService(endpoint,
if err != nil {
return nil, err
}
// Check if bucket exists
exists, err := svc.bucketExists(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
if err := ensureS3Bucket(svc); err != nil {
return nil, err
}
if !exists {
if err = svc.createBucket(context.Background()); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
}
return svc, nil
}
@ -124,18 +114,30 @@ func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, reg
if err != nil {
return nil, err
}
exists, err := svc.bucketExists(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
if err = svc.createBucket(context.Background()); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
if err := ensureS3Bucket(svc); err != nil {
return nil, err
}
return svc, nil
}
func ensureS3Bucket(svc *s3FileService) error {
headCtx, cancel := objectStorageSetupContext()
exists, err := svc.bucketExists(headCtx)
cancel()
if err != nil {
return fmt.Errorf("failed to check bucket: %w", err)
}
if exists {
return nil
}
createCtx, cancel := objectStorageSetupContext()
defer cancel()
if err := svc.createBucket(createCtx); err != nil {
return fmt.Errorf("failed to create bucket: %w", err)
}
return nil
}
// bucketExists checks if the bucket exists
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
@ -238,6 +240,9 @@ func (s *s3FileService) SaveFile(ctx context.Context,
contentType = utils.GetContentTypeByExt(ext)
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
// Upload file to S3
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
@ -260,15 +265,17 @@ func (s *s3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCl
return nil, err
}
ctx, cancel := objectStorageTransferContext(ctx)
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
})
if err != nil {
cancel()
return nil, fmt.Errorf("failed to get file from S3: %w", err)
}
return resp.Body, nil
return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile deletes a file
@ -278,6 +285,9 @@ func (s *s3FileService) DeleteFile(ctx context.Context, filePath string) error {
return err
}
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
@ -306,6 +316,9 @@ func (s *s3FileService) CopyFile(ctx context.Context,
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
// (url.PathEscape would turn them into %2F and break the bucket/key split).
// srcKey is already validated by parseS3FilePath -> SafeObjectKey.
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(s.bucketName),
CopySource: aws.String(s.bucketName + "/" + srcKey),
@ -330,6 +343,9 @@ func (s *s3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uin
ext := filepath.Ext(safeName)
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
ctx, cancel := objectStorageTransferContext(ctx)
defer cancel()
// Upload bytes to S3
reader := bytes.NewReader(data)
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{

View File

@ -582,7 +582,11 @@ func registerChatLocalImageResolver(
resourceCatalog interfaces.ResourceCatalog,
) {
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
ctx := context.Background()
// The object storage clients bound connection setup but leave the
// transfer to this context, so give it a deadline: a chat turn must
// not hang on one image whose download stalls.
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL)
if err != nil {
return nil, false