diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go index 5bcc0b0e..4ffaa334 100644 --- a/internal/application/service/file/cos.go +++ b/internal/application/service/file/cos.go @@ -33,17 +33,15 @@ type cosFileService struct { const cosScheme = "cos://" func newCOSHTTPClient(secretID, secretKey string) *http.Client { - httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() - client := utils.NewSSRFSafeHTTPClient(httpConfig) + client := objectStorageHTTPClient() // COS GetCredential expects AuthorizationTransport to remain the outermost - // transport. Put the per-request SSRF guard immediately underneath it while - // retaining the safe client's redirect policy. + // transport. The per-request SSRF guard the client came with, over the + // bounded transport, goes immediately underneath it; the client keeps its + // redirect policy. client.Transport = &cos.AuthorizationTransport{ SecretID: secretID, SecretKey: secretKey, - Transport: &utils.SSRFValidatingRoundTripper{ - Base: utils.NewSSRFSafeTransport(httpConfig), - }, + Transport: client.Transport, } return client } @@ -120,6 +118,8 @@ func (s *cosFileService) SaveFile(ctx context.Context, return "", fmt.Errorf("failed to open file: %w", err) } defer src.Close() + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.Object.Put(ctx, objectName, src, nil) if err != nil { return "", fmt.Errorf("failed to upload file to COS: %w", err) @@ -136,11 +136,13 @@ func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.Re if err := utils.SafeObjectKey(objectName); err != nil { return nil, fmt.Errorf("invalid file path: %w", err) } + ctx, cancel := objectStorageTransferContext(ctx) resp, err := s.client.Object.Get(ctx, objectName, nil) if err != nil { + cancel() return nil, fmt.Errorf("failed to get file from COS: %w", err) } - return resp.Body, nil + return objectStorageBoundReader(resp.Body, cancel), nil } // DeleteFile removes a file from COS storage @@ -152,6 +154,8 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error if err := utils.SafeObjectKey(objectName); err != nil { return fmt.Errorf("invalid file path: %w", err) } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.Object.Delete(ctx, objectName) if err != nil { return fmt.Errorf("failed to delete file: %w", err) @@ -209,6 +213,9 @@ func (s *cosFileService) CopyFile(ctx context.Context, destKey := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext) // sourceURL is the host + object key WITHOUT a scheme, per the COS SDK contract. + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + sourceURL := fmt.Sprintf("%s.cos.%s.myqcloud.com/%s", s.bucketName, s.region, srcObjectKey) _, _, err = s.client.Object.Copy(ctx, destKey, sourceURL, nil) if err != nil { @@ -229,9 +236,11 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui return "", fmt.Errorf("invalid file name: %w", err) } ext := filepath.Ext(safeName) + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() reader := bytes.NewReader(data) - // 如果请求写入临时桶且临时桶已配置 + // ???????????????? if temp && s.tempClient != nil { objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext) _, err := s.tempClient.Object.Put(ctx, objectName, reader, nil) @@ -242,7 +251,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil } - // 写入主桶 + // ???? objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext) _, err = s.client.Object.Put(ctx, objectName, reader, nil) if err != nil { @@ -254,7 +263,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui // GetFileURL returns a presigned download URL for the file func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { - // 判断文件属于哪个桶 + // ????????? if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) { objectName := strings.TrimPrefix(filePath, s.tempBucketURL) if err := utils.SafeObjectKey(objectName); err != nil { diff --git a/internal/application/service/file/ks3.go b/internal/application/service/file/ks3.go index 8954fbcd..2fbe4cd8 100644 --- a/internal/application/service/file/ks3.go +++ b/internal/application/service/file/ks3.go @@ -3,9 +3,11 @@ package file import ( "bytes" "context" + "errors" "fmt" "io" "mime/multipart" + "net/http" "path/filepath" "strings" "time" @@ -15,6 +17,7 @@ import ( "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" ks3aws "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" "github.com/ks3sdklib/aws-sdk-go/aws/credentials" ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3" ) @@ -65,20 +68,26 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err S3ForcePathStyle: false, // KS3 uses virtual-hosted style SignerVersion: "V2", // KS3 recommends V2 signing MaxRetries: 3, - HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), + HTTPClient: objectStorageHTTPClient(), }) return client, nil } func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error { - _, err := client.HeadBucket(&ks3s3.HeadBucketInput{ + headCtx, cancel := objectStorageSetupContext() + _, err := client.HeadBucketWithContext(headCtx, &ks3s3.HeadBucketInput{ Bucket: ks3aws.String(bucketName), }) + cancel() if err == nil { return nil } - // Bucket doesn't exist, try to create it - _, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{ + if !isKS3BucketMissing(err) { + return fmt.Errorf("failed to check KS3 bucket %q: %w", bucketName, err) + } + createCtx, cancel := objectStorageSetupContext() + defer cancel() + _, createErr := client.CreateBucketWithContext(createCtx, &ks3s3.CreateBucketInput{ Bucket: ks3aws.String(bucketName), }) if createErr != nil { @@ -87,6 +96,30 @@ func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error { return nil } +func isKS3BucketMissing(err error) bool { + if err == nil { + return false + } + var rf awserr.RequestFailure + if errors.As(err, &rf) { + if rf.StatusCode() == http.StatusNotFound { + return true + } + switch rf.Code() { + case "NoSuchBucket", "NotFound": + return true + } + } + var ae awserr.Error + if errors.As(err, &ae) { + switch ae.Code() { + case "NoSuchBucket", "NotFound": + return true + } + } + return false +} + // CheckKS3Connectivity tests KS3 connectivity using the provided credentials. func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { client, err := newKS3Client(endpoint, region, accessKey, secretKey) @@ -94,20 +127,12 @@ func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secr return err } - done := make(chan error, 1) - go func() { - _, err := client.HeadBucket(&ks3s3.HeadBucketInput{ - Bucket: ks3aws.String(bucketName), - }) - done <- err - }() - - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-done: - return err - } + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{ + Bucket: ks3aws.String(bucketName), + }) + return err } func joinKS3Key(parts ...string) string { @@ -161,7 +186,9 @@ func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeade contentType = utils.GetContentTypeByExt(ext) } - _, err = s.client.PutObject(&ks3s3.PutObjectInput{ + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{ Bucket: ks3aws.String(s.bucketName), Key: ks3aws.String(objectKey), Body: src, @@ -182,7 +209,9 @@ func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID ui ext := filepath.Ext(safeName) objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext) - _, err = s.client.PutObject(&ks3s3.PutObjectInput{ + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{ Bucket: ks3aws.String(s.bucketName), Key: ks3aws.String(objectKey), Body: bytes.NewReader(data), @@ -215,7 +244,9 @@ func (s *ks3FileService) CopyFile(ctx context.Context, ext := filepath.Ext(srcPath) destKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext) - _, err = s.client.CopyObject(&ks3s3.CopyObjectInput{ + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.CopyObjectWithContext(ctx, &ks3s3.CopyObjectInput{ Bucket: ks3aws.String(s.bucketName), Key: ks3aws.String(destKey), SourceBucket: ks3aws.String(s.bucketName), @@ -239,15 +270,17 @@ func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadC return nil, fmt.Errorf("invalid file path: %w", err) } - resp, err := s.client.GetObject(&ks3s3.GetObjectInput{ + ctx, cancel := objectStorageTransferContext(ctx) + resp, err := s.client.GetObjectWithContext(ctx, &ks3s3.GetObjectInput{ Bucket: ks3aws.String(s.bucketName), Key: ks3aws.String(objectKey), }) if err != nil { + cancel() return nil, fmt.Errorf("failed to get file from KS3: %w", err) } - return resp.Body, nil + return objectStorageBoundReader(resp.Body, cancel), nil } func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error { @@ -259,7 +292,9 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error return fmt.Errorf("invalid file path: %w", err) } - _, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{ + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.DeleteObjectWithContext(ctx, &ks3s3.DeleteObjectInput{ Bucket: ks3aws.String(s.bucketName), Key: ks3aws.String(objectKey), }) @@ -270,19 +305,12 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error } func (s *ks3FileService) CheckConnectivity(ctx context.Context) error { - done := make(chan error, 1) - go func() { - _, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{ - Bucket: ks3aws.String(s.bucketName), - }) - done <- err - }() - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-done: - return err - } + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err := s.client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{ + Bucket: ks3aws.String(s.bucketName), + }) + return err } func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) { diff --git a/internal/application/service/file/object_storage_http.go b/internal/application/service/file/object_storage_http.go new file mode 100644 index 00000000..5c15b35d --- /dev/null +++ b/internal/application/service/file/object_storage_http.go @@ -0,0 +1,100 @@ +package file + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/utils" +) + +const ( + // objectStorageSetupTimeout bounds the bucket probe a file service runs + // when it is built. Services are rebuilt per resolution, so the probe + // runs before many transfers, not just at startup. + objectStorageSetupTimeout = 30 * time.Second + + // objectStorageTransferTimeout is the fallback when the caller did not + // set a deadline (asynq workers and the main HTTP server both omit one). + // Long enough for a slow 40 MB COS PUT; short enough that a stalled body + // cannot pin a worker forever. + objectStorageTransferTimeout = 30 * time.Minute + + // objectStorageResponseHeaderTimeout is longer than a typical GET/PUT + // header wait because CopyObject / Object.Copy are synchronous: the + // server does not send headers until the copy finishes. GET headers + // still arrive immediately, so a silent peer fails in this window + // rather than the full transfer budget. + objectStorageResponseHeaderTimeout = 10 * time.Minute +) + +// objectStorageHTTPClientConfig is the SSRF-safe client config for the object +// storage SDKs. The default config's Timeout is Go's whole-request timeout, +// body included, which capped every transfer at what a link moves in +// 30 seconds (#3306). The object storage clients bound connection setup on +// their transport instead and leave the transfer to the request context. +func objectStorageHTTPClientConfig() utils.SSRFSafeHTTPClientConfig { + config := utils.DefaultSSRFSafeHTTPClientConfig() + config.Timeout = 0 + return config +} + +// objectStorageTransport is the SSRF-safe transport with the setup bounds the +// whole-request timeout used to provide: a peer that accepts the connection +// and then goes silent fails within the header wait instead of holding the +// request until its context ends. The header wait starts after the request +// body is written, so it does not cap an upload. +func objectStorageTransport() *http.Transport { + transport := utils.NewSSRFSafeTransport(objectStorageHTTPClientConfig()) + transport.TLSHandshakeTimeout = 10 * time.Second + transport.ResponseHeaderTimeout = objectStorageResponseHeaderTimeout + transport.ExpectContinueTimeout = time.Second + transport.IdleConnTimeout = 90 * time.Second + return transport +} + +// objectStorageHTTPClient is the client for the SDKs that take a whole +// http.Client (S3, KS3, OBS, OSS). COS wraps objectStorageTransport in its +// own signing transport. +func objectStorageHTTPClient() *http.Client { + return utils.NewSSRFSafeHTTPClientWithTransport(objectStorageHTTPClientConfig(), objectStorageTransport()) +} + +// objectStorageSetupContext bounds one constructor probe (HeadBucket or +// CreateBucket). Callers must not reuse the same context for both: a slow +// Head that then reports "missing" would leave Create with no time left. +func objectStorageSetupContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), objectStorageSetupTimeout) +} + +// objectStorageTransferContext adds a fallback deadline only when the caller +// did not set one, matching models/chat.withLLMTimeout. +func objectStorageTransferContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + if _, ok := ctx.Deadline(); ok { + return ctx, func() {} + } + return context.WithTimeout(ctx, objectStorageTransferTimeout) +} + +type objectStorageBoundReadCloser struct { + io.ReadCloser + cancel context.CancelFunc +} + +func objectStorageBoundReader(body io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { + if body == nil { + cancel() + return nil + } + return &objectStorageBoundReadCloser{ReadCloser: body, cancel: cancel} +} + +func (r *objectStorageBoundReadCloser) Close() error { + err := r.ReadCloser.Close() + r.cancel() + return err +} diff --git a/internal/application/service/file/object_storage_http_test.go b/internal/application/service/file/object_storage_http_test.go new file mode 100644 index 00000000..eeb55725 --- /dev/null +++ b/internal/application/service/file/object_storage_http_test.go @@ -0,0 +1,136 @@ +package file + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" + "github.com/stretchr/testify/require" + "github.com/tencentyun/cos-go-sdk-v5" + + "github.com/Tencent/WeKnora/internal/utils" +) + +// The object storage clients must not carry Go's whole-request timeout: it +// bounds the body transfer too, so it capped every upload and download at +// what a link can move in 30 seconds (#3306). +func TestObjectStorageHTTPClientConfigDropsOnlyTheWholeRequestTimeout(t *testing.T) { + want := utils.DefaultSSRFSafeHTTPClientConfig() + want.Timeout = 0 + + require.Equal(t, want, objectStorageHTTPClientConfig()) +} + +// Without the whole-request timeout, connection setup is bounded on the +// transport instead; a silent peer must still fail in bounded time. +func TestObjectStorageTransportBoundsConnectionSetup(t *testing.T) { + transport := objectStorageTransport() + + require.Equal(t, 10*time.Second, transport.TLSHandshakeTimeout) + require.Equal(t, objectStorageResponseHeaderTimeout, transport.ResponseHeaderTimeout) + require.Equal(t, time.Second, transport.ExpectContinueTimeout) + require.Equal(t, 90*time.Second, transport.IdleConnTimeout) + require.NotNil(t, transport.DialContext, "the SSRF-safe dialer must stay in place") +} + +func TestObjectStorageHTTPClientKeepsTheSSRFGuard(t *testing.T) { + client := objectStorageHTTPClient() + + require.Zero(t, client.Timeout) + _, ok := client.Transport.(*utils.SSRFValidatingRoundTripper) + require.True(t, ok) + require.NotNil(t, client.CheckRedirect) +} + +func TestCOSHTTPClientHasNoWholeRequestTimeout(t *testing.T) { + client := newCOSHTTPClient("id", "key") + + require.Zero(t, client.Timeout) + auth, ok := client.Transport.(*cos.AuthorizationTransport) + require.True(t, ok, "COS signing must stay the outermost transport") + guard, ok := auth.Transport.(*utils.SSRFValidatingRoundTripper) + require.True(t, ok, "the SSRF guard must stay underneath the signer") + base, ok := guard.Base.(*http.Transport) + require.True(t, ok) + require.Equal(t, objectStorageResponseHeaderTimeout, base.ResponseHeaderTimeout) +} + +func TestOBSClientHasNoWholeRequestTimeout(t *testing.T) { + options := obsS3Options("obs.cn-north-4.myhuaweicloud.com", "cn-north-4", "ak", "sk") + + client, ok := options.HTTPClient.(*http.Client) + require.True(t, ok) + require.Zero(t, client.Timeout) +} + +func TestS3ClientHasNoWholeRequestTimeout(t *testing.T) { + svc, err := newS3Client("", "ak", "sk", "bucket", "us-east-1", "", false) + require.NoError(t, err) + + client, ok := svc.client.Options().HTTPClient.(*http.Client) + require.True(t, ok) + require.Zero(t, client.Timeout) +} + +func TestKS3ClientHasNoWholeRequestTimeout(t *testing.T) { + // The endpoint check resolves the host unless it is whitelisted; keep the + // test off the network. + t.Setenv("SSRF_WHITELIST", "ks3-cn-beijing.ksyuncs.com") + utils.ResetSSRFWhitelistForTest() + t.Cleanup(utils.ResetSSRFWhitelistForTest) + client, err := newKS3Client("https://ks3-cn-beijing.ksyuncs.com", "cn-beijing", "ak", "sk") + require.NoError(t, err) + + require.Zero(t, client.Config.HTTPClient.Timeout) +} + +func TestOSSClientHasNoWholeRequestTimeout(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "oss-cn-hangzhou.aliyuncs.com") + utils.ResetSSRFWhitelistForTest() + t.Cleanup(utils.ResetSSRFWhitelistForTest) + client, err := newOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "cn-hangzhou", "ak", "sk") + require.NoError(t, err) + require.NotNil(t, client) + require.Zero(t, objectStorageHTTPClient().Timeout) +} + +func TestObjectStorageTransferContextAddsFallbackDeadline(t *testing.T) { + ctx, cancel := objectStorageTransferContext(context.Background()) + defer cancel() + + deadline, ok := ctx.Deadline() + require.True(t, ok) + remain := time.Until(deadline) + require.Greater(t, remain, 29*time.Minute) + require.Less(t, remain, 31*time.Minute) +} + +func TestObjectStorageTransferContextKeepsCallerDeadline(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ctx, stop := objectStorageTransferContext(parent) + defer stop() + require.Equal(t, parent, ctx) +} + +func TestObjectStorageBoundReaderCancelsOnClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + body := objectStorageBoundReader(io.NopCloser(strings.NewReader("x")), cancel) + require.NoError(t, body.Close()) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +func TestKS3BucketMissingDetection(t *testing.T) { + require.True(t, isKS3BucketMissing(awserr.NewRequestFailure( + awserr.New("NoSuchBucket", "missing", nil), http.StatusNotFound, "req"))) + require.True(t, isKS3BucketMissing(awserr.NewRequestFailure( + awserr.New("NotFound", "missing", nil), http.StatusNotFound, "req"))) + require.False(t, isKS3BucketMissing(awserr.NewRequestFailure( + awserr.New("AccessDenied", "denied", nil), http.StatusForbidden, "req"))) + require.False(t, isKS3BucketMissing(context.DeadlineExceeded)) +} diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.go index bc5df6e8..ecafa3a8 100644 --- a/internal/application/service/file/obs.go +++ b/internal/application/service/file/obs.go @@ -2,6 +2,7 @@ package file import ( "context" + "errors" "fmt" "io" "mime/multipart" @@ -18,6 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" ) @@ -54,7 +56,7 @@ func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Option // trailing checksum negotiation; align with the S3 driver (s3.go). RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, UsePathStyle: obsUsePathStyle(endpoint), - HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), + HTTPClient: objectStorageHTTPClient(), } } @@ -68,15 +70,24 @@ func NewObsFileService( client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey)) - _, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{ + headCtx, cancel := objectStorageSetupContext() + _, err := client.HeadBucket(headCtx, &s3.HeadBucketInput{ Bucket: aws.String(bucketName), }) + cancel() if err != nil { - _, createErr := client.CreateBucket(context.Background(), &s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if createErr != nil { - fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr) + var notFound *s3types.NotFound + if errors.As(err, ¬Found) { + createCtx, createCancel := objectStorageSetupContext() + _, createErr := client.CreateBucket(createCtx, &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + createCancel() + if createErr != nil { + fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr) + } + } else { + fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, err) } } @@ -176,6 +187,8 @@ func (s *obsFileService) SaveFile(ctx context.Context, contentType = "application/octet-stream" } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectKey), @@ -200,15 +213,17 @@ func (s *obsFileService) GetFile(ctx context.Context, filePath string) (io.ReadC return nil, err } + ctx, cancel := objectStorageTransferContext(ctx) output, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectKey), }) if err != nil { + cancel() return nil, fmt.Errorf("failed to get file from OBS: %w", err) } - return output.Body, nil + return objectStorageBoundReader(output.Body, cancel), nil } func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error { @@ -217,6 +232,8 @@ func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error return err } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectKey), @@ -285,6 +302,8 @@ func (s *obsFileService) CopyFile(ctx context.Context, // CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded // (url.PathEscape would turn them into %2F and break the bucket/key split). + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: aws.String(s.bucketName), CopySource: aws.String(s.bucketName + "/" + srcKey), @@ -323,6 +342,8 @@ func (s *obsFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui } } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectKey), diff --git a/internal/application/service/file/oss.go b/internal/application/service/file/oss.go index bde515e6..1f6d57f0 100644 --- a/internal/application/service/file/oss.go +++ b/internal/application/service/file/oss.go @@ -43,14 +43,16 @@ func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, e WithCredentialsProvider(creds). WithRegion(region). WithEndpoint(endpoint). - WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())) + WithHttpClient(objectStorageHTTPClient()) return oss.NewClient(cfg), nil } // ossEnsureBucket checks if the bucket exists and creates it if missing. func ossEnsureBucket(client *oss.Client, bucketName string) error { - exists, err := client.IsBucketExist(context.Background(), bucketName) + headCtx, cancel := objectStorageSetupContext() + exists, err := client.IsBucketExist(headCtx, bucketName) + cancel() if err != nil { return fmt.Errorf("failed to check OSS bucket: %w", err) } @@ -58,7 +60,9 @@ func ossEnsureBucket(client *oss.Client, bucketName string) error { return nil } - _, err = client.PutBucket(context.Background(), &oss.PutBucketRequest{ + createCtx, cancel := objectStorageSetupContext() + defer cancel() + _, err = client.PutBucket(createCtx, &oss.PutBucketRequest{ Bucket: oss.Ptr(bucketName), }) if err != nil { @@ -123,7 +127,9 @@ func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secr return err } - exists, err := client.IsBucketExist(ctx, bucketName) + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + exists, err := client.IsBucketExist(checkCtx, bucketName) if err != nil { return fmt.Errorf("failed to check OSS bucket: %w", err) } @@ -180,6 +186,9 @@ func (s *ossFileService) SaveFile(ctx context.Context, contentType = utils.GetContentTypeByExt(ext) } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + // Use Uploader for files > 10MB (auto multipart with concurrent uploads) const multipartThreshold = 10 * 1024 * 1024 if file.Size > multipartThreshold { @@ -234,6 +243,8 @@ func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui objectName = fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext) } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = client.PutObject(ctx, &oss.PutObjectRequest{ Bucket: oss.Ptr(targetBucket), Key: oss.Ptr(objectName), @@ -264,6 +275,8 @@ func (s *ossFileService) CopyFile(ctx context.Context, ext := filepath.Ext(srcPath) destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext) + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = s.client.CopyObject(ctx, &oss.CopyObjectRequest{ Bucket: oss.Ptr(s.bucketName), Key: oss.Ptr(destKey), @@ -296,15 +309,17 @@ func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadC client = s.client } + ctx, cancel := objectStorageTransferContext(ctx) resp, err := client.GetObject(ctx, &oss.GetObjectRequest{ Bucket: oss.Ptr(bucketName), Key: oss.Ptr(objectName), }) if err != nil { + cancel() return nil, fmt.Errorf("failed to get file from OSS: %w", err) } - return resp.Body, nil + return objectStorageBoundReader(resp.Body, cancel), nil } // DeleteFile removes a file from OSS. @@ -324,6 +339,8 @@ func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error client = s.client } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() _, err = client.DeleteObject(ctx, &oss.DeleteObjectRequest{ Bucket: oss.Ptr(bucketName), Key: oss.Ptr(objectName), diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go index f43bb3b5..b405e2a2 100644 --- a/internal/application/service/file/s3.go +++ b/internal/application/service/file/s3.go @@ -58,7 +58,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix // Create S3 client with custom endpoint if provided. // For S3-compatible services (non-AWS), use path-style addressing // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key). - httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()) + httpClient := objectStorageHTTPClient() var client *s3.Client if endpoint != "" { usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com") @@ -101,19 +101,9 @@ func NewS3FileService(endpoint, if err != nil { return nil, err } - - // Check if bucket exists - exists, err := svc.bucketExists(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to check bucket: %w", err) + if err := ensureS3Bucket(svc); err != nil { + return nil, err } - - if !exists { - if err = svc.createBucket(context.Background()); err != nil { - return nil, fmt.Errorf("failed to create bucket: %w", err) - } - } - return svc, nil } @@ -124,18 +114,30 @@ func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, reg if err != nil { return nil, err } - exists, err := svc.bucketExists(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to check bucket: %w", err) - } - if !exists { - if err = svc.createBucket(context.Background()); err != nil { - return nil, fmt.Errorf("failed to create bucket: %w", err) - } + if err := ensureS3Bucket(svc); err != nil { + return nil, err } return svc, nil } +func ensureS3Bucket(svc *s3FileService) error { + headCtx, cancel := objectStorageSetupContext() + exists, err := svc.bucketExists(headCtx) + cancel() + if err != nil { + return fmt.Errorf("failed to check bucket: %w", err) + } + if exists { + return nil + } + createCtx, cancel := objectStorageSetupContext() + defer cancel() + if err := svc.createBucket(createCtx); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) + } + return nil +} + // bucketExists checks if the bucket exists func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) { _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{ @@ -238,6 +240,9 @@ func (s *s3FileService) SaveFile(ctx context.Context, contentType = utils.GetContentTypeByExt(ext) } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + // Upload file to S3 _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucketName), @@ -260,15 +265,17 @@ func (s *s3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCl return nil, err } + ctx, cancel := objectStorageTransferContext(ctx) resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectName), }) if err != nil { + cancel() return nil, fmt.Errorf("failed to get file from S3: %w", err) } - return resp.Body, nil + return objectStorageBoundReader(resp.Body, cancel), nil } // DeleteFile deletes a file @@ -278,6 +285,9 @@ func (s *s3FileService) DeleteFile(ctx context.Context, filePath string) error { return err } + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(objectName), @@ -306,6 +316,9 @@ func (s *s3FileService) CopyFile(ctx context.Context, // CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded // (url.PathEscape would turn them into %2F and break the bucket/key split). // srcKey is already validated by parseS3FilePath -> SafeObjectKey. + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + _, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: aws.String(s.bucketName), CopySource: aws.String(s.bucketName + "/" + srcKey), @@ -330,6 +343,9 @@ func (s *s3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uin ext := filepath.Ext(safeName) objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext) + ctx, cancel := objectStorageTransferContext(ctx) + defer cancel() + // Upload bytes to S3 reader := bytes.NewReader(data) _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ diff --git a/internal/container/container.go b/internal/container/container.go index 16bd7df0..a1d429a6 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -582,7 +582,11 @@ func registerChatLocalImageResolver( resourceCatalog interfaces.ResourceCatalog, ) { chat.LocalImageResolver = func(storageURL string) ([]byte, bool) { - ctx := context.Background() + // The object storage clients bound connection setup but leave the + // transfer to this context, so give it a deadline: a chat turn must + // not hang on one image whose download stalls. + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL) if err != nil { return nil, false