diff --git a/.env.example b/.env.example index c1e874d7..b9876e98 100644 --- a/.env.example +++ b/.env.example @@ -717,10 +717,10 @@ LANGFUSE_HOST=http://langfuse-web:3000 # # ===== 向量库地址的 SSRF 校验 ===== # 通过 API 创建/测试向量库连接时,地址会经过 SSRF 校验。docker-compose 内置服务 -# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。 +# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng, minio)已通过 SSRF_WHITELIST_EXTRA 默认放行。 # 如使用外部私网地址或被拦截端口(5432, 9200 等),请将主机加入 SSRF_WHITELIST_EXTRA # (注意:自定义该变量会覆盖 compose 默认值,需重新包含内置服务名),例如: -# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,my-opensearch.internal +# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio,my-opensearch.internal # # Doris Stream Load:FE 会将请求 307 重定向到 BE;docker compose 默认放行 # doris-fe 与 doris-be(见 SSRF_WHITELIST_EXTRA)。 diff --git a/docker-compose.yml b/docker-compose.yml index 51550382..1ecb490b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -285,17 +285,17 @@ services: - SSRF_WHITELIST=${SSRF_WHITELIST:-} # 保留原始 URL 的图片域名白名单(逗号分隔,不替换为 provider://) - IMAGE_HOST_KEEP_URL=${IMAGE_HOST_KEEP_URL:-} - # Always allow the optional searxng sidecar and the bundled vector-store - # service hostnames (compose service names). Vector-store create/test now - # runs SSRF validation on user-supplied addresses, which would otherwise - # reject these private docker-network hosts; whitelisting them keeps the + # Always allow the optional searxng sidecar, bundled vector stores, and + # the default MinIO storage host (compose service names). User-controlled + # endpoints are SSRF-validated and these private docker-network hosts + # would otherwise be rejected; whitelisting them keeps the # out-of-box experience. Merged on top of SSRF_WHITELIST so user # overrides don't clobber it. (postgres is intentionally NOT listed: it # is not a user-registerable vector store, and whitelisting the app's own # DB host would skip the 5432 port block.) # doris-be: Stream Load follows FE→BE redirects that must receive Basic # credentials on whitelisted BE hosts. - - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be} + - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} # File size limit (in MB) diff --git a/docreader/parser/opendataloader_parser.py b/docreader/parser/opendataloader_parser.py index f6374c4f..bdb96c28 100644 --- a/docreader/parser/opendataloader_parser.py +++ b/docreader/parser/opendataloader_parser.py @@ -25,6 +25,7 @@ from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.base_parser import BaseParser from docreader.parser.concurrency import parser_worker_limit +from docreader.utils.ssrf import is_ssrf_safe_url logger = logging.getLogger(__name__) @@ -34,6 +35,13 @@ _MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") _IMAGE_FILE_NUM_RE = re.compile(r"^imageFile(\d+)\.", re.I) +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Health probes never need redirects; refusing them closes redirect SSRF.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + def _override_str(overrides: Optional[Mapping[str, Any]], key: str, default: str = "") -> str: if overrides: v = overrides.get(key) @@ -75,11 +83,16 @@ def _ping_hybrid( base = url.rstrip("/") health_url = f"{base}/health" + safe, reason = is_ssrf_safe_url(health_url) + if not safe: + return False, f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}" + + opener = urllib.request.build_opener(_NoRedirectHandler()) last_err = "" for attempt in range(max(1, retries)): try: req = urllib.request.Request(health_url, method="GET") - with urllib.request.urlopen(req, timeout=timeout_sec) as resp: + with opener.open(req, timeout=timeout_sec) as resp: if 200 <= resp.status < 300: return True, "" last_err = f"hybrid 健康检查 HTTP {resp.status}: {health_url}" @@ -278,6 +291,11 @@ def _run_convert( kwargs["hybrid"] = hybrid hybrid_url = _resolve_hybrid_url(overrides) if hybrid_url: + safe, reason = is_ssrf_safe_url(hybrid_url) + if not safe: + raise RuntimeError( + f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}" + ) kwargs["hybrid_url"] = hybrid_url hybrid_mode = _override_str(overrides, "odl_hybrid_mode", CONFIG.odl_hybrid_mode) if hybrid_mode: diff --git a/docreader/tests/test_opendataloader_parser.py b/docreader/tests/test_opendataloader_parser.py index 28a6f974..70792dcd 100644 --- a/docreader/tests/test_opendataloader_parser.py +++ b/docreader/tests/test_opendataloader_parser.py @@ -1,6 +1,7 @@ """Unit tests for OpenDataLoader parser helpers (no JVM required).""" import os +import sys import tempfile import unittest from unittest import mock @@ -10,12 +11,41 @@ from docreader.parser.opendataloader_parser import ( _collect_images_under_output, _find_markdown_file, _normalize_odl_image_url, + _ping_hybrid, + _run_convert, _rewrite_markdown_image_refs, opendataloader_available, ) class OpenDataLoaderHelpersTest(unittest.TestCase): + def test_hybrid_health_probe_blocks_private_url_before_request(self): + with mock.patch( + "docreader.parser.opendataloader_parser.is_ssrf_safe_url", + return_value=(False, "restricted test address"), + ), mock.patch("urllib.request.build_opener") as build_opener: + ok, msg = _ping_hybrid("http://127.0.0.1:8080", retries=1) + + self.assertFalse(ok) + self.assertIn("SSRF", msg) + build_opener.assert_not_called() + + def test_convert_blocks_private_hybrid_url_at_final_sink(self): + fake_module = mock.Mock() + with mock.patch.dict(sys.modules, {"opendataloader_pdf": fake_module}): + with self.assertRaisesRegex(RuntimeError, "SSRF"): + _run_convert( + "/tmp/input.pdf", + "/tmp/output", + "/tmp/output/images", + overrides={ + "odl_hybrid": "docling-fast", + "odl_hybrid_url": "http://169.254.169.254/latest/meta-data", + }, + ) + + fake_module.convert.assert_not_called() + def test_find_markdown_prefers_stem_match(self): with tempfile.TemporaryDirectory() as d: other = os.path.join(d, "other.md") diff --git a/docreader/tests/test_ssrf.py b/docreader/tests/test_ssrf.py index d42b6468..a4f944db 100644 --- a/docreader/tests/test_ssrf.py +++ b/docreader/tests/test_ssrf.py @@ -1,3 +1,4 @@ +import ipaddress import os import unittest from unittest.mock import patch @@ -36,6 +37,20 @@ class TestSSRFValidation(unittest.TestCase): self.assertFalse(safe) self.assertTrue(reason) + def test_blocks_invalid_port(self): + safe, reason = is_ssrf_safe_url("https://example.com:99999/path") + self.assertFalse(safe) + self.assertIn("invalid port", reason) + + def test_blocks_ipv4_mapped_private_ipv6_resolution(self): + with patch( + "docreader.utils.ssrf._resolve_host_ips", + return_value=((ipaddress.ip_address("::ffff:127.0.0.1"),), None), + ): + safe, reason = is_ssrf_safe_url("https://example.invalid/path") + self.assertFalse(safe) + self.assertIn("restricted", reason) + def test_allows_public_https(self): safe, reason = is_ssrf_safe_url("https://example.com/article") self.assertTrue(safe, reason) diff --git a/docreader/utils/ssrf.py b/docreader/utils/ssrf.py index c13f6a41..be33e004 100644 --- a/docreader/utils/ssrf.py +++ b/docreader/utils/ssrf.py @@ -161,6 +161,21 @@ def _is_restricted_ip(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) - if isinstance(net, ipaddress.IPv4Network) and ip in net: return f"restricted range {net}" if isinstance(ip, ipaddress.IPv6Address): + embedded_ipv4 = ip.ipv4_mapped + if embedded_ipv4 is not None: + reason = _is_restricted_ip(embedded_ipv4) + if reason: + return f"IPv4-mapped {reason}" + if ip.sixtofour is not None: + reason = _is_restricted_ip(ip.sixtofour) + if reason: + return f"6to4-embedded {reason}" + if ip.teredo is not None: + server_ip, client_ip = ip.teredo + for label, embedded_ip in (("server", server_ip), ("client", client_ip)): + reason = _is_restricted_ip(embedded_ip) + if reason: + return f"Teredo {label} embeds {reason}" # Site-local (fec0::/10) if (ip.packed[0] == 0xFE) and (ip.packed[1] & 0xC0) == 0xC0: return "site-local IPv6 address" @@ -238,7 +253,10 @@ def is_ssrf_safe_url(raw_url: str) -> Tuple[bool, str]: f"hostname {hostname_lower} resolves to restricted IP {resolved_ip}: {reason}", ) - port = parsed.port + try: + port = parsed.port + except ValueError as exc: + return False, f"invalid port: {exc}" if port is not None and str(port) in BLOCKED_PORTS: return False, f"port {port} is blocked for security reasons" diff --git a/internal/application/repository/retriever/opensearch/ssrf_test.go b/internal/application/repository/retriever/opensearch/ssrf_test.go new file mode 100644 index 00000000..790c9e87 --- /dev/null +++ b/internal/application/repository/retriever/opensearch/ssrf_test.go @@ -0,0 +1,16 @@ +package opensearch + +import ( + "os" + "testing" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + // End-to-end repository tests intentionally use loopback httptest servers. + secutils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost,opensearch.example.com") + code := m.Run() + secutils.SetSSRFWhitelistFromRaw("") + os.Exit(code) +} diff --git a/internal/application/repository/retriever/opensearch/transport.go b/internal/application/repository/retriever/opensearch/transport.go index c39fda27..fbe0e1b6 100644 --- a/internal/application/repository/retriever/opensearch/transport.go +++ b/internal/application/repository/retriever/opensearch/transport.go @@ -10,6 +10,7 @@ import ( osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // NewOpenSearchClient builds a TLS-hardened, pool-tuned *osapi.Client for @@ -35,13 +36,16 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) { if cfg == nil || cfg.Addr == "" { return nil, fmt.Errorf("opensearch: ConnectionConfig.Addr required: %w", ErrConfigInvalid) } + if err := secutils.ValidateURLForSSRF(cfg.Addr); err != nil { + return nil, fmt.Errorf("opensearch: address failed SSRF validation: %w", err) + } transport := buildHTTPTransport(cfg.InsecureSkipVerify) return osapi.NewClient(osapi.Config{ Client: opensearch.Config{ Addresses: []string{cfg.Addr}, Username: cfg.Username, Password: cfg.Password, - Transport: transport, + Transport: &secutils.SSRFValidatingRoundTripper{Base: transport}, }, }) } @@ -54,6 +58,8 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) { // final client fragile across SDK versions). func buildHTTPTransport(insecureSkipVerify bool) *http.Transport { return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: secutils.SSRFSafeDialContext, TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, InsecureSkipVerify: insecureSkipVerify, //nolint:gosec — operator opt-in flag diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go index 6427bd8f..442584f2 100644 --- a/internal/application/service/file/cos.go +++ b/internal/application/service/file/cos.go @@ -32,6 +32,22 @@ type cosFileService struct { const cosScheme = "cos://" +func newCOSHTTPClient(secretID, secretKey string) *http.Client { + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() + client := utils.NewSSRFSafeHTTPClient(httpConfig) + // COS GetCredential expects AuthorizationTransport to remain the outermost + // transport. Put the per-request SSRF guard immediately underneath it while + // retaining the safe client's redirect policy. + client.Transport = &cos.AuthorizationTransport{ + SecretID: secretID, + SecretKey: secretKey, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }, + } + return client +} + // newCosClient creates a bare cosFileService with just the SDK client initialised. // Shared by NewCosFileService* constructors and CheckCosConnectivity. func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) { @@ -41,12 +57,7 @@ func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileServi if err != nil { return nil, fmt.Errorf("failed to parse bucketURL: %w", err) } - client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: secretID, - SecretKey: secretKey, - }, - }) + client := cos.NewClient(&cos.BaseURL{BucketURL: u}, newCOSHTTPClient(secretID, secretKey)) return &cosFileService{client: client, bucketURL: bucketURL, bucketName: bucketName, region: region}, nil } @@ -72,12 +83,7 @@ func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, co if err != nil { return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err) } - svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: secretId, - SecretKey: secretKey, - }, - }) + svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, newCOSHTTPClient(secretId, secretKey)) svc.tempBucketURL = tempBucketURL } diff --git a/internal/application/service/file/endpoint_security_test.go b/internal/application/service/file/endpoint_security_test.go new file mode 100644 index 00000000..55e6d09f --- /dev/null +++ b/internal/application/service/file/endpoint_security_test.go @@ -0,0 +1,41 @@ +package file + +import ( + "context" + "testing" +) + +func TestStorageClientsRejectUnsafeEndpointAtConstruction(t *testing.T) { + const endpoint = "http://169.254.169.254/latest/meta-data" + tests := map[string]func() error{ + "minio": func() error { + _, err := newMinioClient(endpoint, "ak", "sk", "bucket", false) + return err + }, + "s3": func() error { + _, err := newS3Client(endpoint, "ak", "sk", "bucket", "region", "", true) + return err + }, + "obs": func() error { + return CheckObsConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket") + }, + "oss": func() error { + _, err := newOSSClient(endpoint, "region", "ak", "sk") + return err + }, + "tos": func() error { + return CheckTosConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket") + }, + "ks3": func() error { + _, err := newKS3Client(endpoint, "region", "ak", "sk") + return err + }, + } + for name, run := range tests { + t.Run(name, func(t *testing.T) { + if err := run(); err == nil { + t.Fatal("expected unsafe endpoint to be rejected") + } + }) + } +} diff --git a/internal/application/service/file/ks3.go b/internal/application/service/file/ks3.go index f4489915..07b5528c 100644 --- a/internal/application/service/file/ks3.go +++ b/internal/application/service/file/ks3.go @@ -53,6 +53,9 @@ func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathP } func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe KS3 endpoint: %w", err) + } creds := credentials.NewStaticCredentials(accessKey, secretKey, "") client := ks3s3.New(&ks3aws.Config{ Credentials: creds, @@ -62,6 +65,7 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err S3ForcePathStyle: false, // KS3 uses virtual-hosted style SignerVersion: "V2", // KS3 recommends V2 signing MaxRetries: 3, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) return client, nil } diff --git a/internal/application/service/file/minio.go b/internal/application/service/file/minio.go index 7ef51d4f..7ba43235 100644 --- a/internal/application/service/file/minio.go +++ b/internal/application/service/file/minio.go @@ -28,9 +28,16 @@ type minioFileService struct { // Shared by NewMinioFileService (which also ensures the bucket exists) and // CheckMinioConnectivity (read-only probe). func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe MinIO endpoint: %w", err) + } + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() client, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }, }) if err != nil { return nil, fmt.Errorf("failed to initialize MinIO client: %w", err) diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.go index 1b8ffe8e..451d92db 100644 --- a/internal/application/service/file/obs.go +++ b/internal/application/service/file/obs.go @@ -12,6 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -43,12 +44,16 @@ func NewObsFileService( endpoint, region, accessKeyID, secretAccessKey, bucketName string, pathPrefix string, ) (interfaces.FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe OBS endpoint: %w", err) + } client := s3.New(s3.Options{ Region: region, EndpointResolver: &obsEndpointResolver{url: endpoint}, Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""), UsePathStyle: true, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) _, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{ @@ -76,6 +81,9 @@ func NewObsFileService( } func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("unsafe OBS endpoint: %w", err) + } checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -84,6 +92,7 @@ func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secr EndpointResolver: &obsEndpointResolver{url: endpoint}, Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), UsePathStyle: true, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) _, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{ diff --git a/internal/application/service/file/oss.go b/internal/application/service/file/oss.go index 48974c77..bde515e6 100644 --- a/internal/application/service/file/oss.go +++ b/internal/application/service/file/oss.go @@ -34,12 +34,16 @@ const ossScheme = "oss://" // newOSSClient creates an OSS client using the official Aliyun SDK v2. func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe OSS endpoint: %w", err) + } creds := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") cfg := oss.LoadDefaultConfig(). WithCredentialsProvider(creds). WithRegion(region). - WithEndpoint(endpoint) + WithEndpoint(endpoint). + WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())) return oss.NewClient(cfg), nil } diff --git a/internal/application/service/file/oss_test.go b/internal/application/service/file/oss_test.go index 4144f29e..679f9402 100644 --- a/internal/application/service/file/oss_test.go +++ b/internal/application/service/file/oss_test.go @@ -117,7 +117,7 @@ func TestNewOSSClient(t *testing.T) { }, { name: "custom endpoint", - endpoint: "https://custom-oss-endpoint.com", + endpoint: "https://example.com", region: "cn-shanghai", accessKey: "ak", secretKey: "sk", @@ -145,6 +145,12 @@ func TestNewOSSClient(t *testing.T) { } } +func TestNewOSSClientRejectsUnsafeEndpoint(t *testing.T) { + if _, err := newOSSClient("http://127.0.0.1:9000", "cn-hangzhou", "ak", "sk"); err == nil { + t.Fatal("expected loopback OSS endpoint to be rejected") + } +} + func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go index ffd40c9d..02da6a02 100644 --- a/internal/application/service/file/s3.go +++ b/internal/application/service/file/s3.go @@ -31,6 +31,9 @@ type s3FileService struct { // newS3Client creates a bare s3FileService with just the SDK client initialised. func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe S3 endpoint: %w", err) + } var cfg aws.Config var err error @@ -55,16 +58,20 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix // Create S3 client with custom endpoint if provided. // For S3-compatible services (non-AWS), use path-style addressing // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key). + httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()) var client *s3.Client if endpoint != "" { usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com") client = s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpoint) o.UsePathStyle = usePathStyle + o.HTTPClient = httpClient }) } else { // Standard AWS S3 - client = s3.NewFromConfig(cfg) + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.HTTPClient = httpClient + }) } // Normalize pathPrefix: ensure it ends with '/' if not empty diff --git a/internal/application/service/file/tos.go b/internal/application/service/file/tos.go index 39f5f344..a260f7f3 100644 --- a/internal/application/service/file/tos.go +++ b/internal/application/service/file/tos.go @@ -36,10 +36,17 @@ func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathP // NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket. func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe TOS endpoint: %w", err) + } + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() client, err := tos.NewClientV2( endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }), ) if err != nil { return nil, fmt.Errorf("failed to initialize TOS client: %w", err) @@ -58,6 +65,9 @@ func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, buc endpoint, tos.WithRegion(tempRegion), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }), ) if err != nil { return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err) @@ -87,10 +97,16 @@ func (s *tosFileService) CheckConnectivity(ctx context.Context) error { // CheckTosConnectivity tests TOS connectivity using the provided credentials. func CheckTosConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("unsafe TOS endpoint: %w", err) + } client, err := tos.NewClientV2( endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }), ) if err != nil { return fmt.Errorf("failed to initialize TOS client: %w", err) diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 64b9eb68..fb539633 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3593,6 +3593,16 @@ func (s *knowledgeService) convert( } mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides) applyParserRuleOverrides(mergedOverrides, eff.ChunkingConfig, fileType) + if err := validateParserEngineOverrideURLs(mergedOverrides); err != nil { + logger.Errorf(ctx, "Parser endpoint rejected for SSRF protection: %v", err) + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "Parser endpoint is not allowed for security reasons" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + s.failStage(ctx, knowledge.ID, types.StageDocReader, + werrors.ErrCodeDocReaderParseFailed, knowledge.ErrorMessage, err) + return nil, nil + } if isURL { if err := secutils.ValidateURLForSSRF(payload.URL); err != nil { diff --git a/internal/application/service/knowledge_util_ssrf_test.go b/internal/application/service/knowledge_util_ssrf_test.go index c3e8d0a9..6a2a4df1 100644 --- a/internal/application/service/knowledge_util_ssrf_test.go +++ b/internal/application/service/knowledge_util_ssrf_test.go @@ -33,7 +33,9 @@ func TestDownloadFileFromURLBlocksRedirectToLoopback(t *testing.T) { if err == nil { t.Fatal("expected redirect to loopback to be blocked") } - if !strings.Contains(err.Error(), "redirect blocked") && !strings.Contains(err.Error(), "connection blocked") { + if !strings.Contains(err.Error(), "redirect blocked") && + !strings.Contains(err.Error(), "connection blocked") && + !strings.Contains(err.Error(), "outbound request blocked") { t.Fatalf("unexpected error: %v", err) } } diff --git a/internal/application/service/mcp_service.go b/internal/application/service/mcp_service.go index 9aadfd74..8899641d 100644 --- a/internal/application/service/mcp_service.go +++ b/internal/application/service/mcp_service.go @@ -41,6 +41,9 @@ func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types if service.TransportType == types.MCPTransportStdio { return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead") } + if err := mcp.ValidateServiceOutboundURLs(service); err != nil { + return err + } // Set default advanced config if not provided if service.AdvancedConfig == nil { @@ -248,6 +251,9 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types } // Update timestamp + if err := mcp.ValidateServiceOutboundURLs(existing); err != nil { + return err + } existing.UpdatedAt = time.Now() if err := s.mcpServiceRepo.Update(ctx, existing); err != nil { diff --git a/internal/application/service/parser_url_security.go b/internal/application/service/parser_url_security.go new file mode 100644 index 00000000..cb2d3db3 --- /dev/null +++ b/internal/application/service/parser_url_security.go @@ -0,0 +1,33 @@ +package service + +import ( + "fmt" + "strings" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +var parserOutboundURLKeys = []string{ + "mineru_endpoint", + "mineru_vlm_server_url", + "odl_hybrid_url", + "paddleocr_vl_endpoint", + "paddleocr_vl_cloud_base_url", +} + +// validateParserEngineOverrideURLs validates every parser override that can +// cause this process or the trusted DocReader service to make an outbound +// request. Per-upload overrides are included because API callers can provide +// the generic parser_engine_overrides map directly. +func validateParserEngineOverrideURLs(overrides map[string]string) error { + for _, key := range parserOutboundURLKeys { + rawURL := strings.TrimSpace(overrides[key]) + if rawURL == "" { + continue + } + if err := secutils.ValidateURLForSSRF(rawURL); err != nil { + return fmt.Errorf("%s failed SSRF validation: %w", key, err) + } + } + return nil +} diff --git a/internal/application/service/parser_url_security_test.go b/internal/application/service/parser_url_security_test.go new file mode 100644 index 00000000..87c1efe8 --- /dev/null +++ b/internal/application/service/parser_url_security_test.go @@ -0,0 +1,24 @@ +package service + +import "testing" + +func TestValidateParserOverrideURLsRejectsEverySupportedURLKey(t *testing.T) { + for _, key := range parserOutboundURLKeys { + t.Run(key, func(t *testing.T) { + err := validateParserEngineOverrideURLs(map[string]string{ + key: "http://169.254.169.254/latest/meta-data", + }) + if err == nil { + t.Fatalf("expected %s to be rejected", key) + } + }) + } +} + +func TestValidateParserOverrideURLsIgnoresNonURLKeys(t *testing.T) { + if err := validateParserEngineOverrideURLs(map[string]string{ + "mineru_model_version": "pipeline", + }); err != nil { + t.Fatalf("unexpected error for non-URL override: %v", err) + } +} diff --git a/internal/application/service/vectorstore_healthcheck.go b/internal/application/service/vectorstore_healthcheck.go index 36eadee9..44f1caf9 100644 --- a/internal/application/service/vectorstore_healthcheck.go +++ b/internal/application/service/vectorstore_healthcheck.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "net" "net/http" "strings" "time" @@ -15,6 +14,7 @@ import ( "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/go-sql-driver/mysql" // MySQL driver for database/sql, used by Doris connection test _ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql "github.com/qdrant/go-client/qdrant" @@ -22,6 +22,7 @@ import ( "github.com/weaviate/weaviate-go-client/v5/weaviate" "github.com/weaviate/weaviate-go-client/v5/weaviate/auth" wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc" + "google.golang.org/grpc" ) const connectionTestTimeout = 10 * time.Second @@ -71,14 +72,14 @@ func testElasticsearchConnection(ctx context.Context, config types.ConnectionCon req.SetBasicAuth(config.Username, config.Password) } - client := &http.Client{ - Timeout: connectionTestTimeout, + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = connectionTestTimeout + client := secutils.NewSSRFSafeHTTPClient(clientCfg) + client.CheckRedirect = func(*http.Request, []*http.Request) error { // A health probe must not follow redirects: a malicious or compromised // endpoint could return a 302 to an internal address, defeating the // SSRF check that only validated the original (user-supplied) Addr. - CheckRedirect: func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse - }, + return http.ErrUseLastResponse } resp, err := client.Do(req) if err != nil { @@ -155,6 +156,9 @@ func testQdrantConnection(ctx context.Context, config types.ConnectionConfig) (s Port: port, APIKey: config.APIKey, UseTLS: config.UseTLS, + GrpcOptions: []grpc.DialOption{ + grpc.WithContextDialer(secutils.SSRFSafeGRPCDialer), + }, }) if err != nil { return "", errors.NewBadRequestError("failed to create qdrant client: invalid configuration") @@ -184,7 +188,7 @@ func testMilvusConnection(ctx context.Context, config types.ConnectionConfig) (s addr = "localhost:19530" } - conn, err := (&net.Dialer{}).DialContext(testCtx, "tcp", addr) + conn, err := secutils.SSRFSafeDialContext(testCtx, "tcp", addr) if err != nil { logger.Warnf(ctx, "Milvus connection test failed: %v", err) return "", errors.NewBadRequestError("failed to connect to milvus: connection refused or server unreachable") @@ -201,6 +205,9 @@ func testTencentVectorDBConnection(ctx context.Context, config types.ConnectionC client, err := tcvectordb.NewRpcClient(config.Addr, config.Username, config.APIKey, &tcvectordb.ClientOption{ ReadConsistency: tcvectordb.EventualConsistency, Timeout: connectionTestTimeout, + Transport: &secutils.SSRFValidatingRoundTripper{ + Base: secutils.NewSSRFSafeTransport(secutils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { logger.Warnf(ctx, "Tencent VectorDB connection test failed: %v", err) @@ -233,7 +240,8 @@ func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig) } weaviateCfg := weaviate.Config{ - Host: host, + Host: host, + ConnectionClient: secutils.NewSSRFSafeHTTPClient(secutils.DefaultSSRFSafeHTTPClientConfig()), GrpcConfig: &wgrpc.Config{ Host: grpcAddress, }, @@ -291,7 +299,8 @@ func testDorisConnection(ctx context.Context, config types.ConnectionConfig) (st cfg := mysql.NewConfig() cfg.User = config.Username cfg.Passwd = config.Password - cfg.Net = "tcp" + secutils.RegisterMySQLSSRFDialer() + cfg.Net = secutils.MySQLSSRFNetwork cfg.Addr = config.Addr cfg.DBName = database cfg.Timeout = 5 * time.Second diff --git a/internal/application/service/vectorstore_healthcheck_ssrf_test.go b/internal/application/service/vectorstore_healthcheck_ssrf_test.go new file mode 100644 index 00000000..2e23cf00 --- /dev/null +++ b/internal/application/service/vectorstore_healthcheck_ssrf_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestTestConnection_MilvusBlocksUnsafeAddrAtDialSink(t *testing.T) { + svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := svc.TestConnection(ctx, types.MilvusRetrieverEngineType, types.ConnectionConfig{ + Addr: "169.254.169.254:19530", + }) + if err == nil { + t.Fatal("expected milvus connectivity probe to reject unsafe address") + } + if !strings.Contains(err.Error(), "failed to connect to milvus") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestTestConnection_DorisBlocksUnsafeAddrAtDialSink(t *testing.T) { + svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := svc.TestConnection(ctx, types.DorisRetrieverEngineType, types.ConnectionConfig{ + Addr: "169.254.169.254:9030", + Database: "weknora", + Username: "root", + }) + if err == nil { + t.Fatal("expected doris connectivity probe to reject unsafe address") + } + if !strings.Contains(err.Error(), "failed to connect to doris") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/application/service/weknoracloud.go b/internal/application/service/weknoracloud.go index a6d6ce6e..0c838602 100644 --- a/internal/application/service/weknoracloud.go +++ b/internal/application/service/weknoracloud.go @@ -72,7 +72,9 @@ func (s *weKnoraCloudService) verifyCredentials(ctx context.Context, appID, appS logger.Infof(ctx, "credential verification request: method=GET url=%s app_id=%s request_id=%s ", healthURL, appID, requestID) - client := &http.Client{Timeout: 10 * time.Second} + clientCfg := utils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 10 * time.Second + client := utils.NewSSRFSafeHTTPClient(clientCfg) resp, err := client.Do(req) if err != nil { logger.Warnf(ctx, "credential verification HTTP failed: url=%s err=%v", healthURL, err) diff --git a/internal/container/engine_factory.go b/internal/container/engine_factory.go index 1567a8b7..c37f757d 100644 --- a/internal/container/engine_factory.go +++ b/internal/container/engine_factory.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "net" "strconv" "strings" "time" @@ -33,6 +34,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/tencent/vectordatabase-sdk-go/tcvectordb" ) @@ -58,6 +60,9 @@ func createEngineServiceFromStore( cfg *config.Config, auditSink openSearchRepo.AuditSink, ) (interfaces.RetrieveEngineService, error) { + if err := validateRuntimeVectorStoreAddresses(store); err != nil { + return nil, err + } switch store.EngineType { case types.PostgresRetrieverEngineType: return createPostgresEngine(store, db) @@ -82,6 +87,48 @@ func createEngineServiceFromStore( } } +// validateRuntimeVectorStoreAddresses is the final guard for persisted or +// imported vector-store rows. Create/test handlers validate the same fields at +// the input boundary, but runtime construction must not assume every stored row +// was written through those handlers. +func validateRuntimeVectorStoreAddresses(store types.VectorStore) error { + cc := store.ConnectionConfig + check := func(label, endpoint string) error { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return nil + } + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("%s failed SSRF validation: %w", label, err) + } + return nil + } + + switch store.EngineType { + case types.PostgresRetrieverEngineType, types.SQLiteRetrieverEngineType: + return nil + case types.ElasticsearchRetrieverEngineType, + types.OpenSearchRetrieverEngineType, + types.MilvusRetrieverEngineType, + types.TencentVectorDBRetrieverEngineType, + types.DorisRetrieverEngineType: + return check("vector store address", cc.Addr) + case types.QdrantRetrieverEngineType: + endpoint := cc.Host + if endpoint != "" && cc.Port != 0 { + endpoint = net.JoinHostPort(strings.Trim(cc.Host, "[]"), strconv.Itoa(cc.Port)) + } + return check("qdrant address", endpoint) + case types.WeaviateRetrieverEngineType: + if err := check("weaviate HTTP address", cc.Host); err != nil { + return err + } + return check("weaviate gRPC address", cc.GrpcAddress) + default: + return fmt.Errorf("vector store engine %q has no SSRF address policy", store.EngineType) + } +} + // createOpenSearchEngine builds an OpenSearch k-NN retrieve engine. Mirrors // createElasticsearchV8Engine but uses the driver's TLS-hardened client // constructor and injects the audit sink. NewRepository probes the cluster @@ -142,10 +189,16 @@ func isESv7(version string) bool { func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) { cc := store.ConnectionConfig + if err := utils.ValidateURLForSSRF(cc.Addr); err != nil { + return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err) + } client, err := elasticsearch.NewTypedClient(elasticsearch.Config{ Addresses: []string{cc.Addr}, Username: cc.Username, Password: cc.Password, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create elasticsearch v8 client: %w", err) @@ -156,10 +209,16 @@ func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (i func createElasticsearchV7Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) { cc := store.ConnectionConfig + if err := utils.ValidateURLForSSRF(cc.Addr); err != nil { + return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err) + } client, err := esv7.NewClient(esv7.Config{ Addresses: []string{cc.Addr}, Username: cc.Username, Password: cc.Password, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create elasticsearch v7 client: %w", err) @@ -176,10 +235,11 @@ func createQdrantEngine(store types.VectorStore) (interfaces.RetrieveEngineServi } client, err := qdrant.NewClient(&qdrant.Config{ - Host: cc.Host, - Port: port, - APIKey: cc.APIKey, - UseTLS: cc.UseTLS, + Host: cc.Host, + Port: port, + APIKey: cc.APIKey, + UseTLS: cc.UseTLS, + GrpcOptions: []grpc.DialOption{grpc.WithContextDialer(utils.SSRFSafeGRPCDialer)}, }) if err != nil { return nil, fmt.Errorf("create qdrant client: %w", err) @@ -205,8 +265,11 @@ func buildMilvusClientConfig(cc types.ConnectionConfig) milvusclient.ClientConfi } milvusCfg := milvusclient.ClientConfig{ - Address: addr, - DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)}, + Address: addr, + DialOptions: []grpc.DialOption{ + grpc.WithTimeout(5 * time.Second), + grpc.WithContextDialer(utils.SSRFSafeGRPCDialer), + }, } if cc.Username != "" { milvusCfg.Username = cc.Username @@ -236,7 +299,8 @@ func createWeaviateEngine(store types.VectorStore) (interfaces.RetrieveEngineSer } weaviateCfg := weaviate.Config{ - Host: host, + Host: host, + ConnectionClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), GrpcConfig: &wgrpc.Config{ Host: grpcAddress, }, @@ -275,7 +339,8 @@ func createDorisEngine(store types.VectorStore) (interfaces.RetrieveEngineServic mc := mysql.NewConfig() mc.User = cc.Username mc.Passwd = cc.Password - mc.Net = "tcp" + utils.RegisterMySQLSSRFDialer() + mc.Net = utils.MySQLSSRFNetwork mc.Addr = cc.Addr mc.DBName = cc.Database mc.Params = map[string]string{"charset": "utf8mb4"} @@ -314,6 +379,9 @@ func createTencentVectorDBEngine(store types.VectorStore) (interfaces.RetrieveEn client, err := tcvectordb.NewRpcClient(cc.Addr, cc.Username, cc.APIKey, &tcvectordb.ClientOption{ ReadConsistency: tcvectordb.EventualConsistency, Timeout: 10 * time.Second, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create tencent vectordb client: %w", err) diff --git a/internal/container/engine_factory_test.go b/internal/container/engine_factory_test.go index bd61c418..6114df4a 100644 --- a/internal/container/engine_factory_test.go +++ b/internal/container/engine_factory_test.go @@ -26,8 +26,8 @@ func TestBuildMilvusClientConfig_UsesDatabaseName(t *testing.T) { if cfg.DBName != "regdi_ram_haom1" { t.Fatalf("expected DBName to be propagated, got %q", cfg.DBName) } - if len(cfg.DialOptions) != 1 { - t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions)) + if len(cfg.DialOptions) != 2 { + t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions)) } } @@ -40,7 +40,7 @@ func TestBuildMilvusClientConfig_DefaultsAddressWhenMissing(t *testing.T) { if cfg.DBName != "" { t.Fatalf("expected empty DBName by default, got %q", cfg.DBName) } - if len(cfg.DialOptions) != 1 { - t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions)) + if len(cfg.DialOptions) != 2 { + t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions)) } } diff --git a/internal/container/ssrf_test.go b/internal/container/ssrf_test.go new file mode 100644 index 00000000..69173a50 --- /dev/null +++ b/internal/container/ssrf_test.go @@ -0,0 +1,43 @@ +package container + +import ( + "os" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + // Engine wiring tests intentionally use loopback httptest servers. + utils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost") + code := m.Run() + utils.SetSSRFWhitelistFromRaw("") + os.Exit(code) +} + +func TestValidateRuntimeVectorStoreAddressesRejectsUnsafeEndpoints(t *testing.T) { + tests := []types.VectorStore{ + {EngineType: types.QdrantRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "169.254.169.254", Port: 6334}}, + {EngineType: types.MilvusRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:19530"}}, + {EngineType: types.WeaviateRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "https://example.com", GrpcAddress: "169.254.169.254:50051"}}, + {EngineType: types.DorisRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:9030"}}, + {EngineType: types.TencentVectorDBRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:80"}}, + } + for _, store := range tests { + if err := validateRuntimeVectorStoreAddresses(store); err == nil { + t.Fatalf("expected %s unsafe address to be rejected", store.EngineType) + } + } +} + +func TestValidateRuntimeVectorStoreAddressesAllowsNonNetworkEngines(t *testing.T) { + for _, engineType := range []types.RetrieverEngineType{ + types.PostgresRetrieverEngineType, + types.SQLiteRetrieverEngineType, + } { + if err := validateRuntimeVectorStoreAddresses(types.VectorStore{EngineType: engineType}); err != nil { + t.Fatalf("unexpected %s validation error: %v", engineType, err) + } + } +} diff --git a/internal/handler/mcp_service.go b/internal/handler/mcp_service.go index 9d3255f4..55919eb2 100644 --- a/internal/handler/mcp_service.go +++ b/internal/handler/mcp_service.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/handler/dto" "github.com/Tencent/WeKnora/internal/logger" + mcpsecurity "github.com/Tencent/WeKnora/internal/mcp" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -74,6 +75,11 @@ func (h *MCPServiceHandler) CreateMCPService(c *gin.Context) { return } } + if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil { + logger.Warnf(ctx, "SSRF validation failed for MCP service configuration: %v", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } if err := h.mcpServiceService.CreateMCPService(ctx, &service); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_name": secutils.SanitizeForLog(service.Name)}) @@ -333,6 +339,11 @@ func (h *MCPServiceHandler) UpdateMCPService(c *gin.Context) { service.AdvancedConfig.RetryDelay = int(retryDelay) } } + if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil { + logger.Warnf(ctx, "SSRF validation failed for MCP service update: %v", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } if err := h.mcpServiceService.UpdateMCPService(ctx, &service); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)}) diff --git a/internal/handler/system.go b/internal/handler/system.go index c55e6cf4..69ff853d 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -28,8 +28,6 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "github.com/neo4j/neo4j-go-driver/v6/neo4j" ) @@ -863,59 +861,16 @@ func storageEndpointHost(endpoint string) string { return endpoint } -// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous -// address (cloud metadata, loopback, link-local). Unlike the stricter isSSRFSafeURL, -// this allows private IPs since MinIO is commonly deployed on internal networks. -// It also respects the SSRF_WHITELIST environment variable for whitelisted hosts. +// isBlockedStorageEndpoint keeps the legacy handler contract while delegating +// to the same fail-closed SSRF policy used by the storage clients themselves. +// Private storage endpoints must be explicitly whitelisted by an operator. func isBlockedStorageEndpoint(endpoint string) (bool, string) { - host := storageEndpointHost(endpoint) - if host == "" { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { return true, "无效的地址" } - - // Check SSRF whitelist first – whitelisted hosts bypass the block check. - if secutils.IsSSRFWhitelisted(host) { - return false, "" - } - - hostLower := strings.ToLower(host) - - blockedHosts := []string{ - "metadata.google.internal", - "metadata.tencentyun.com", - "metadata.aws.internal", - } - for _, bh := range blockedHosts { - if hostLower == bh { - return true, "该地址不允许访问" - } - } - - checkIP := func(ip net.IP) (bool, string) { - if ip.IsLoopback() { - return true, "不允许访问本地回环地址" - } - if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return true, "不允许访问链路本地地址" - } - if ip.IsUnspecified() { - return true, "无效的地址" - } - return false, "" - } - - if ip := net.ParseIP(host); ip != nil { - return checkIP(ip) - } - - ips, err := net.LookupIP(host) - if err != nil { - return false, "" - } - for _, ip := range ips { - if blocked, reason := checkIP(ip); blocked { - return blocked, reason - } + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return true, secutils.FormatSSRFError("存储 Endpoint", endpoint, err) } return false, "" } @@ -1029,15 +984,7 @@ func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *typ // If bucket does not exist, auto-create it if strings.Contains(errMsg, "does not exist") && cfg.BucketName != "" { logger.Info(ctx, "Storage check: bucket does not exist, attempting auto-creation", "bucket", cfg.BucketName) - minioClient, clientErr := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: cfg.UseSSL, - }) - if clientErr != nil { - c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to create MinIO client: %s", sanitizeStorageCheckError(clientErr))}}) - return - } - if mkErr := minioClient.MakeBucket(ctx, cfg.BucketName, minio.MakeBucketOptions{}); mkErr != nil { + if _, mkErr := file.NewMinioFileService(endpoint, accessKeyID, secretAccessKey, cfg.BucketName, cfg.UseSSL); mkErr != nil { logger.Error(ctx, "Storage check: failed to create bucket", "bucket", cfg.BucketName, "error", mkErr) c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to auto-create Bucket '%s': %s", cfg.BucketName, sanitizeStorageCheckError(mkErr))}}) return diff --git a/internal/handler/tenant.go b/internal/handler/tenant.go index 5f92a329..9bc99927 100644 --- a/internal/handler/tenant.go +++ b/internal/handler/tenant.go @@ -1737,5 +1737,15 @@ func validateParserEngineOutboundURLs(cfg *types.ParserEngineConfig) error { return fmt.Errorf("mineru_vlm_server_url failed SSRF validation: %v", err) } } + if odlURL := strings.TrimSpace(cfg.ODLHybridURL); odlURL != "" { + if err := secutils.ValidateURLForSSRF(odlURL); err != nil { + return fmt.Errorf("odl_hybrid_url failed SSRF validation: %v", err) + } + } + if endpoint := strings.TrimSpace(cfg.PaddleOCRVLEndpoint); endpoint != "" { + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("paddleocr_vl_endpoint failed SSRF validation: %v", err) + } + } return nil } diff --git a/internal/im/dingtalk/adapter.go b/internal/im/dingtalk/adapter.go index db574015..a39c45d0 100644 --- a/internal/im/dingtalk/adapter.go +++ b/internal/im/dingtalk/adapter.go @@ -26,7 +26,10 @@ import ( ) // httpClient is a shared HTTP client with a reasonable timeout for DingTalk API calls. -var httpClient = &http.Client{Timeout: 15 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 15 * time.Second, + MaxRedirects: 5, +}) // apiBaseURL is the DingTalk OpenAPI host. Overridable in tests. var apiBaseURL = "https://api.dingtalk.com" @@ -420,6 +423,9 @@ func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, r } func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, content string) error { + if err := secutils.ValidateURLForSSRF(webhookURL); err != nil { + return fmt.Errorf("dingtalk sessionWebhook rejected by SSRF policy: %w", err) + } body := map[string]interface{}{ "msgtype": "markdown", "markdown": map[string]string{ diff --git a/internal/im/dingtalk/download_test.go b/internal/im/dingtalk/download_test.go index a3a8f6d7..1eeb5dee 100644 --- a/internal/im/dingtalk/download_test.go +++ b/internal/im/dingtalk/download_test.go @@ -7,15 +7,24 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/Tencent/WeKnora/internal/im" ) +func useTestHTTPClient(t *testing.T) { + t.Helper() + original := httpClient + httpClient = &http.Client{Timeout: 5 * time.Second} + t.Cleanup(func() { httpClient = original }) +} + // TestDownloadFile_EndToEnd drives the full DownloadFile orchestration against a // fake DingTalk OpenAPI: access-token fetch → messageFiles/download (downloadCode // → temporary downloadUrl) → GET the temp URL for the bytes. This covers the HTTP // path that the pure-function unit tests deliberately skip (issue #1771). func TestDownloadFile_EndToEnd(t *testing.T) { + useTestHTTPClient(t) fileBytes := []byte("%PDF-1.7 fake product spec bytes") var downloadReq map[string]string @@ -89,6 +98,7 @@ func TestDownloadFile_EndToEnd(t *testing.T) { // TestDownloadFile_TempURLError verifies a non-200 from the temporary download // URL surfaces as an error rather than silently returning empty content. func TestDownloadFile_TempURLError(t *testing.T) { + useTestHTTPClient(t) var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -121,6 +131,7 @@ func TestDownloadFile_TempURLError(t *testing.T) { } func TestDownloadFile_SSRFRejected(t *testing.T) { + useTestHTTPClient(t) var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { diff --git a/internal/im/feishu/adapter.go b/internal/im/feishu/adapter.go index 0b744954..bc5902fb 100644 --- a/internal/im/feishu/adapter.go +++ b/internal/im/feishu/adapter.go @@ -39,7 +39,10 @@ import ( var _ im.StreamSender = (*Adapter)(nil) var _ im.FileDownloader = (*Adapter)(nil) -var httpClient = &http.Client{Timeout: 10 * time.Second} +var httpClient = utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ + Timeout: 10 * time.Second, + MaxRedirects: 5, +}) // Adapter implements im.Adapter for Feishu/Lark. type Adapter struct { diff --git a/internal/im/feishu/send_e2e_test.go b/internal/im/feishu/send_e2e_test.go index 4322519b..03aab714 100644 --- a/internal/im/feishu/send_e2e_test.go +++ b/internal/im/feishu/send_e2e_test.go @@ -8,10 +8,18 @@ import ( "strings" "sync" "testing" + "time" "github.com/Tencent/WeKnora/internal/im" ) +func useTestHTTPClient(t *testing.T) { + t.Helper() + original := httpClient + httpClient = &http.Client{Timeout: 5 * time.Second} + t.Cleanup(func() { httpClient = original }) +} + // testRegion returns a Lark-shaped region pointed at a fake Open Platform, so // the whole send path can be driven without touching the real cloud. Region // carries the host as a field precisely so this is possible. @@ -86,6 +94,7 @@ func (f *fakeOpenPlatform) sawPath(want string) bool { // SendReply must fetch a token from the region's cloud and reply under the // original message, carrying that token as a bearer credential. func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{} srv := httptest.NewServer(fake.handler()) defer srv.Close() @@ -130,6 +139,7 @@ func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) { // A group that rejects reply-in-thread (230071) must still receive the answer // via the plain send-message API, addressed to the chat. func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{replyCode: 230071} srv := httptest.NewServer(fake.handler()) defer srv.Close() @@ -160,6 +170,7 @@ func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) { // A non-fallback-eligible error must surface, not be silently swallowed. func TestSendReply_EndToEnd_HardErrorSurfaces(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{replyCode: 99991663} // app ticket invalid — not retryable srv := httptest.NewServer(fake.handler()) defer srv.Close() diff --git a/internal/im/qqbot/longconn.go b/internal/im/qqbot/longconn.go index ca79501e..7e468c3c 100644 --- a/internal/im/qqbot/longconn.go +++ b/internal/im/qqbot/longconn.go @@ -9,6 +9,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ws "github.com/gorilla/websocket" ) @@ -68,7 +69,9 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error { if err != nil { return err } - conn, _, err := ws.DefaultDialer.DialContext(ctx, gatewayURL, nil) + dialer := *ws.DefaultDialer + dialer.NetDialContext = secutils.SSRFSafeDialContext + conn, _, err := dialer.DialContext(ctx, gatewayURL, nil) if err != nil { return err } diff --git a/internal/im/telegram/adapter.go b/internal/im/telegram/adapter.go index cbb0d042..99859469 100644 --- a/internal/im/telegram/adapter.go +++ b/internal/im/telegram/adapter.go @@ -17,6 +17,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // Compile-time checks. @@ -72,8 +73,8 @@ func (a *Adapter) VerifyCallback(c *gin.Context) error { // telegramUpdate represents an incoming Telegram update (subset of fields). type telegramUpdate struct { - UpdateID int `json:"update_id"` - Message *telegramMsg `json:"message"` + UpdateID int `json:"update_id"` + Message *telegramMsg `json:"message"` } type telegramMsg struct { @@ -256,7 +257,10 @@ func (a *Adapter) editMessage(ctx context.Context, chatID, messageID, text, pars } // httpClient is a shared HTTP client with a reasonable timeout for Telegram API calls. -var httpClient = &http.Client{Timeout: 15 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 15 * time.Second, + MaxRedirects: 5, +}) // callAPI calls the Telegram Bot API, discarding the result. func (a *Adapter) callAPI(ctx context.Context, method string, body interface{}) error { @@ -311,7 +315,7 @@ const minEditInterval = 500 * time.Millisecond type streamState struct { mu sync.Mutex content strings.Builder - msgID string // Telegram message ID of the "thinking" message + msgID string // Telegram message ID of the "thinking" message chatID string lastEdit time.Time // last successful editMessageText timestamp createdAt time.Time // for orphan stream detection diff --git a/internal/im/telegram/longconn.go b/internal/im/telegram/longconn.go index 36373677..7d85d5e5 100644 --- a/internal/im/telegram/longconn.go +++ b/internal/im/telegram/longconn.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // MessageHandler is called when an IM message is received via long polling. @@ -26,9 +27,12 @@ type LongConnClient struct { // NewLongConnClient creates a Telegram long-polling client. func NewLongConnClient(botToken string, handler MessageHandler) *LongConnClient { return &LongConnClient{ - botToken: botToken, - handler: handler, - httpClient: &http.Client{Timeout: 35 * time.Second}, + botToken: botToken, + handler: handler, + httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 35 * time.Second, + MaxRedirects: 5, + }), } } diff --git a/internal/im/wechat/adapter.go b/internal/im/wechat/adapter.go index 4a3c3fdb..52411069 100644 --- a/internal/im/wechat/adapter.go +++ b/internal/im/wechat/adapter.go @@ -24,6 +24,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" ) @@ -37,7 +38,10 @@ const ( channelVersion = "weknora-1.0.0" ) -var ilinkHTTPClient = &http.Client{Timeout: 30 * time.Second} +var ilinkHTTPClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 30 * time.Second, + MaxRedirects: 5, +}) // BuildCDNDownloadURL constructs a CDN download URL from an encrypt_query_param. func BuildCDNDownloadURL(encryptQueryParam string) string { @@ -145,6 +149,9 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io if msg.FileKey == "" { return nil, "", fmt.Errorf("no file URL in message") } + if err := secutils.ValidateURLForSSRF(msg.FileKey); err != nil { + return nil, "", fmt.Errorf("file URL rejected by SSRF policy: %w", err) + } fileName := msg.FileName if fileName == "" { diff --git a/internal/im/wechat/longpoll.go b/internal/im/wechat/longpoll.go index b7358c3b..e892279e 100644 --- a/internal/im/wechat/longpoll.go +++ b/internal/im/wechat/longpoll.go @@ -22,13 +22,14 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) const ( - longPollTimeout = 35 * time.Second - longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout - reconnectBaseDelay = 1 * time.Second - reconnectMaxDelay = 30 * time.Second + longPollTimeout = 35 * time.Second + longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout + reconnectBaseDelay = 1 * time.Second + reconnectMaxDelay = 30 * time.Second maxReconnectAttempts = -1 // infinite ) @@ -50,7 +51,10 @@ func NewLongPollClient(botToken, ilinkBotID string, handler func(ctx context.Con botToken: botToken, ilinkBotID: ilinkBotID, handler: handler, - httpClient: &http.Client{Timeout: longPollHTTPTimeout}, + httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: longPollHTTPTimeout, + MaxRedirects: 5, + }), } } @@ -316,25 +320,25 @@ func pollReconnectDelay(attempt int) time.Duration { // ── iLink API response types (matches proto: GetUpdatesResp, WeixinMessage) ── type getUpdatesResponse struct { - Ret int `json:"ret"` - ErrCode int `json:"errcode"` - ErrMsg string `json:"errmsg"` - Msgs []weixinMessage `json:"msgs"` - GetUpdatesBuf string `json:"get_updates_buf"` + Ret int `json:"ret"` + ErrCode int `json:"errcode"` + ErrMsg string `json:"errmsg"` + Msgs []weixinMessage `json:"msgs"` + GetUpdatesBuf string `json:"get_updates_buf"` } type weixinMessage struct { - Seq int `json:"seq"` - MessageID int64 `json:"message_id"` - FromUserID string `json:"from_user_id"` - ToUserID string `json:"to_user_id"` - ClientID string `json:"client_id"` - CreateTimeMs int64 `json:"create_time_ms"` - SessionID string `json:"session_id"` - MessageType int `json:"message_type"` // 1=USER, 2=BOT - MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH - ItemList []messageItem `json:"item_list"` - ContextToken string `json:"context_token"` + Seq int `json:"seq"` + MessageID int64 `json:"message_id"` + FromUserID string `json:"from_user_id"` + ToUserID string `json:"to_user_id"` + ClientID string `json:"client_id"` + CreateTimeMs int64 `json:"create_time_ms"` + SessionID string `json:"session_id"` + MessageType int `json:"message_type"` // 1=USER, 2=BOT + MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH + ItemList []messageItem `json:"item_list"` + ContextToken string `json:"context_token"` } type messageItem struct { diff --git a/internal/im/wechat/qrcode.go b/internal/im/wechat/qrcode.go index 176de300..3bed7017 100644 --- a/internal/im/wechat/qrcode.go +++ b/internal/im/wechat/qrcode.go @@ -1,8 +1,9 @@ // QR code login flow for WeChat iLink Bot API. // // API endpoints (all relative to the iLink base URL): -// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content} -// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...} +// +// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content} +// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...} // // Flow: // 1. Call GetLoginQRCode to obtain a QR code URL and opaque qrcode token @@ -19,6 +20,8 @@ import ( "net/http" "net/url" "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" ) // QRCodeResult holds the result of requesting a login QR code. @@ -46,7 +49,10 @@ type QRCodeService struct { // NewQRCodeService creates a new QR code service. func NewQRCodeService() *QRCodeService { return &QRCodeService{ - client: &http.Client{Timeout: pollTimeout}, + client: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: pollTimeout, + MaxRedirects: 5, + }), } } @@ -76,7 +82,7 @@ func (s *QRCodeService) GetLoginQRCode(ctx context.Context) (*QRCodeResult, erro } var result struct { - QRCode string `json:"qrcode"` + QRCode string `json:"qrcode"` QRCodeImgContent string `json:"qrcode_img_content"` } if err := json.Unmarshal(body, &result); err != nil { diff --git a/internal/im/wecom/longconn.go b/internal/im/wecom/longconn.go index b545c90d..3ecd6cf4 100644 --- a/internal/im/wecom/longconn.go +++ b/internal/im/wecom/longconn.go @@ -22,6 +22,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ws "github.com/gorilla/websocket" ) @@ -365,7 +366,9 @@ func (c *LongConnClient) sendStreamFrame(incoming *im.IncomingMessage, streamID, } func (c *LongConnClient) connectAndRun(ctx context.Context) error { - conn, _, err := ws.DefaultDialer.DialContext(ctx, c.endpoint, nil) + dialer := *ws.DefaultDialer + dialer.NetDialContext = secutils.SSRFSafeDialContext + conn, _, err := dialer.DialContext(ctx, c.endpoint, nil) if err != nil { return fmt.Errorf("dial: %w", err) } diff --git a/internal/im/wecom/webhook_adapter.go b/internal/im/wecom/webhook_adapter.go index 1166eb85..fe37cdb4 100644 --- a/internal/im/wecom/webhook_adapter.go +++ b/internal/im/wecom/webhook_adapter.go @@ -37,7 +37,10 @@ import ( "github.com/gin-gonic/gin" ) -var httpClient = &http.Client{Timeout: 30 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 30 * time.Second, + MaxRedirects: 5, +}) const defaultAPIBaseURL = "https://qyapi.weixin.qq.com" diff --git a/internal/infrastructure/docparser/http_parser.go b/internal/infrastructure/docparser/http_parser.go index 7d0267d8..aeca68e6 100644 --- a/internal/infrastructure/docparser/http_parser.go +++ b/internal/infrastructure/docparser/http_parser.go @@ -14,6 +14,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" ) const ( @@ -62,16 +63,17 @@ type HTTPDocumentReader struct { } func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) { + baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/") + if baseURL != "" { + if err := secutils.ValidateURLForSSRF(baseURL); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } + } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 5 * time.Minute p := &HTTPDocumentReader{ - baseURL: strings.TrimSuffix(baseURL, "/"), - client: &http.Client{ - Timeout: 5 * time.Minute, - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, - }, + baseURL: baseURL, + client: secutils.NewSSRFSafeHTTPClient(clientCfg), } if p.baseURL != "" { logger.Infof(context.Background(), "INFO: HTTP docreader base URL: %s", p.baseURL) @@ -86,9 +88,15 @@ func (p *HTTPDocumentReader) base() string { } func (p *HTTPDocumentReader) Reconnect(addr string) error { + addr = strings.TrimSuffix(strings.TrimSpace(addr), "/") + if addr != "" { + if err := secutils.ValidateURLForSSRF(addr); err != nil { + return fmt.Errorf("docreader address failed SSRF validation: %w", err) + } + } p.mu.Lock() defer p.mu.Unlock() - p.baseURL = strings.TrimSuffix(addr, "/") + p.baseURL = addr logger.Infof(context.Background(), "INFO: HTTP docreader base URL set to %s", p.baseURL) return nil } @@ -122,6 +130,9 @@ func (p *HTTPDocumentReader) ListEngines(ctx context.Context, overrides map[stri if base == "" { return nil, errNotConnected } + if err := secutils.ValidateURLForSSRF(base); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } body := httpListEnginesRequest{ConfigOverrides: overrides} jsonBody, err := json.Marshal(body) @@ -187,6 +198,9 @@ func (p *HTTPDocumentReader) Read(ctx context.Context, req *types.ReadRequest) ( if base == "" { return nil, errNotConnected } + if err := secutils.ValidateURLForSSRF(base); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } body := httpReadRequest{ FileName: req.FileName, diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go index 9beba038..98cfc013 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go @@ -51,6 +51,9 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques if c.token == "" { return &types.ReadResult{Error: "PaddleOCR-VL Cloud token is not configured"}, nil } + if err := utils.ValidateURLForSSRF(c.baseURL); err != nil { + return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL Cloud base URL blocked by SSRF policy: %v", err)}, nil + } content := req.FileContent if len(content) == 0 { diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go index 5ab7fdcb..313d3b74 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go @@ -44,6 +44,9 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (* if c.endpoint == "" { return &types.ReadResult{Error: "PaddleOCR-VL endpoint is not configured"}, nil } + if err := utils.ValidateURLForSSRF(c.endpoint); err != nil { + return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL endpoint blocked by SSRF policy: %v", err)}, nil + } content := req.FileContent if len(content) == 0 { @@ -157,7 +160,10 @@ func (c *PaddleOCRVLReader) callLayoutParsing( } httpReq.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: paddleOCRVLTimeout} + client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ + Timeout: paddleOCRVLTimeout, + MaxRedirects: 5, + }) resp, err := client.Do(httpReq) if err != nil { return "", nil, fmt.Errorf("HTTP request: %w", err) @@ -265,6 +271,9 @@ func PingPaddleOCRVL(endpoint string) (bool, string) { if endpoint == "" { return false, "未配置 PaddleOCR-VL 端点" } + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return false, fmt.Sprintf("PaddleOCR-VL 端点未通过 SSRF 校验: %v", err) + } client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ Timeout: 5 * time.Second, MaxRedirects: 5, diff --git a/internal/infrastructure/docparser/weknoracloud_http_reader.go b/internal/infrastructure/docparser/weknoracloud_http_reader.go index 866085f7..f19ef710 100644 --- a/internal/infrastructure/docparser/weknoracloud_http_reader.go +++ b/internal/infrastructure/docparser/weknoracloud_http_reader.go @@ -15,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/utils" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" ) @@ -39,20 +40,15 @@ func NewWeKnoraCloudSignedDocumentReader(appID, apiKey string) (*WeKnoraCloudSig if apiKey == "" { return nil, fmt.Errorf("WeKnoraCloud apiKey is required") } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 500 * time.Minute return &WeKnoraCloudSignedDocumentReader{ appID: appID, apiKey: apiKey, initialPollInterval: 500 * time.Millisecond, maxPollInterval: 10 * time.Second, pollTimeout: 20 * time.Minute, - client: &http.Client{ - Timeout: 500 * time.Minute, - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, - }, + client: secutils.NewSSRFSafeHTTPClient(clientCfg), }, nil } diff --git a/internal/infrastructure/web_search/baidu.go b/internal/infrastructure/web_search/baidu.go index 3b2ce698..3a8490e1 100644 --- a/internal/infrastructure/web_search/baidu.go +++ b/internal/infrastructure/web_search/baidu.go @@ -39,8 +39,9 @@ func NewBaiduProvider(params types.WebSearchProviderParameters) (interfaces.WebS if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Baidu provider") } - client := &http.Client{ - Timeout: defaultBaiduTimeout, + client, err := NewSearchHTTPClient(defaultBaiduTimeout, params.ProxyURL) + if err != nil { + return nil, fmt.Errorf("create Baidu HTTP client: %w", err) } return &BaiduProvider{ client: client, diff --git a/internal/infrastructure/web_search/ollama.go b/internal/infrastructure/web_search/ollama.go index c555597a..b4e3fa8f 100644 --- a/internal/infrastructure/web_search/ollama.go +++ b/internal/infrastructure/web_search/ollama.go @@ -35,8 +35,9 @@ func NewOllamaProvider(params types.WebSearchProviderParameters) (interfaces.Web if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Ollama provider") } - client := &http.Client{ - Timeout: defaultOllamaTimeout, + client, err := NewSearchHTTPClient(defaultOllamaTimeout, params.ProxyURL) + if err != nil { + return nil, fmt.Errorf("create Ollama HTTP client: %w", err) } return &OllamaProvider{ client: client, diff --git a/internal/infrastructure/web_search/proxy.go b/internal/infrastructure/web_search/proxy.go index d7bc0689..3dc38816 100644 --- a/internal/infrastructure/web_search/proxy.go +++ b/internal/infrastructure/web_search/proxy.go @@ -19,18 +19,6 @@ func ValidateProxyURL(proxyURL string) error { return utils.ValidateURLForSSRF(proxyURL) } -func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error { - return func(req *http.Request, via []*http.Request) error { - if len(via) >= maxRedirects { - return fmt.Errorf("stopped after %d redirects", maxRedirects) - } - if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil { - return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err) - } - return nil - } -} - // NewSearchHTTPClient builds an http.Client for outbound web search requests. // It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and // redirect validation consistent with utils.NewSSRFSafeHTTPClient. @@ -60,9 +48,6 @@ func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client, } cfg := utils.DefaultSSRFSafeHTTPClientConfig() - return &http.Client{ - Timeout: timeout, - Transport: t, - CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects), - }, nil + cfg.Timeout = timeout + return utils.NewSSRFSafeHTTPClientWithTransport(cfg, t), nil } diff --git a/internal/infrastructure/web_search/proxy_test.go b/internal/infrastructure/web_search/proxy_test.go index 46f318d1..9a2fc121 100644 --- a/internal/infrastructure/web_search/proxy_test.go +++ b/internal/infrastructure/web_search/proxy_test.go @@ -59,10 +59,13 @@ func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) { } func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) { - h := ssrfSafeRedirect(10) + client, err := NewSearchHTTPClient(0, "") + if err != nil { + t.Fatal(err) + } req := &http.Request{URL: mustParse(t, "ftp://evil.com/")} // len(via)==0 so first redirect - err := h(req, nil) + err = client.CheckRedirect(req, nil) if err == nil { t.Fatal("expected error for ftp redirect") } diff --git a/internal/mcp/client.go b/internal/mcp/client.go index 63b2bb94..c49962f9 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -12,6 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/mark3labs/mcp-go/client" "github.com/mark3labs/mcp-go/client/transport" "github.com/mark3labs/mcp-go/mcp" @@ -147,15 +148,22 @@ func asOAuthRequired(err error) *OAuthRequiredError { // NewMCPClient creates a new MCP client based on the transport type func NewMCPClient(config *ClientConfig) (MCPClient, error) { + if config == nil || config.Service == nil { + return nil, fmt.Errorf("MCP client config and service are required") + } + if err := ValidateServiceOutboundURLs(config.Service); err != nil { + return nil, err + } + // Create HTTP client with timeout timeout := 30 * time.Second if config.Service.AdvancedConfig != nil && config.Service.AdvancedConfig.Timeout > 0 { timeout = time.Duration(config.Service.AdvancedConfig.Timeout) * time.Second } - httpClient := &http.Client{ - Timeout: timeout, - } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = timeout + httpClient := secutils.NewSSRFSafeHTTPClient(clientCfg) // Build headers headers := make(map[string]string) diff --git a/internal/mcp/oauth_manager.go b/internal/mcp/oauth_manager.go index 761e5fd1..0791e4aa 100644 --- a/internal/mcp/oauth_manager.go +++ b/internal/mcp/oauth_manager.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/mark3labs/mcp-go/client/transport" "github.com/redis/go-redis/v9" ) @@ -52,12 +53,18 @@ func (m *OAuthManager) newHandler( if service.URL == nil || *service.URL == "" { return nil, fmt.Errorf("MCP service URL is required for OAuth") } + if err := ValidateServiceOutboundURLs(service); err != nil { + return nil, err + } + httpCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + httpCfg.Timeout = 30 * time.Second cfg := transport.OAuthConfig{ RedirectURI: redirectURI, Scopes: service.AuthConfig.Scopes, TokenStore: newDBTokenStore(m.repo, tenantID, principal, service.ID), PKCEEnabled: true, AuthServerMetadataURL: service.AuthConfig.AuthServerMetadataURL, + HTTPClient: secutils.NewSSRFSafeHTTPClient(httpCfg), } if existing, err := m.repo.GetClient(ctx, tenantID, service.ID); err == nil && existing != nil { cfg.ClientID = existing.ClientID diff --git a/internal/mcp/security.go b/internal/mcp/security.go new file mode 100644 index 00000000..d614e3ff --- /dev/null +++ b/internal/mcp/security.go @@ -0,0 +1,36 @@ +package mcp + +import ( + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// ValidateServiceOutboundURLs validates every URL that the MCP transport or +// OAuth discovery flow may contact. It is intentionally called both at +// persistence boundaries and immediately before client construction so stale +// or imported rows cannot bypass the current SSRF policy. +func ValidateServiceOutboundURLs(service *types.MCPService) error { + if service == nil { + return fmt.Errorf("MCP service is required") + } + if service.URL != nil { + serviceURL := strings.TrimSpace(*service.URL) + if serviceURL != "" { + if err := secutils.ValidateURLForSSRF(serviceURL); err != nil { + return fmt.Errorf("MCP service URL failed SSRF validation: %w", err) + } + } + } + if service.AuthConfig != nil { + metadataURL := strings.TrimSpace(service.AuthConfig.AuthServerMetadataURL) + if metadataURL != "" { + if err := secutils.ValidateURLForSSRF(metadataURL); err != nil { + return fmt.Errorf("MCP OAuth metadata URL failed SSRF validation: %w", err) + } + } + } + return nil +} diff --git a/internal/mcp/security_test.go b/internal/mcp/security_test.go new file mode 100644 index 00000000..b79ecc8c --- /dev/null +++ b/internal/mcp/security_test.go @@ -0,0 +1,31 @@ +package mcp + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestValidateServiceOutboundURLsRejectsOAuthMetadataSSRF(t *testing.T) { + serviceURL := "https://example.com/mcp" + err := ValidateServiceOutboundURLs(&types.MCPService{ + URL: &serviceURL, + AuthConfig: &types.MCPAuthConfig{ + AuthServerMetadataURL: "http://169.254.169.254/latest/meta-data", + }, + }) + if err == nil { + t.Fatal("expected OAuth metadata URL to be rejected") + } +} + +func TestNewMCPClientRejectsStoredUnsafeURL(t *testing.T) { + serviceURL := "http://127.0.0.1:8080/mcp" + _, err := NewMCPClient(&ClientConfig{Service: &types.MCPService{ + URL: &serviceURL, + TransportType: types.MCPTransportHTTPStreamable, + }}) + if err == nil { + t.Fatal("expected stored unsafe MCP URL to be rejected at client construction") + } +} diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 9364cb71..02b93d0f 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -72,16 +72,16 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { } } + // The SDK must use the same SSRF-safe transport as the raw HTTP paths. + // Constructor-time URL validation alone cannot prevent DNS rebinding or a + // later redirect to an internal address. + sdkHTTPClient := rawHTTPClient // 如果指定了 CustomHeaders,则给 SDK 使用的 HTTPClient 挂一层 RoundTripper, // 在每个请求上自动注入这些 header(raw HTTP 路径会在发送前单独处理)。 if len(chatConfig.CustomHeaders) > 0 { - if httpClient, ok := config.HTTPClient.(*http.Client); ok { - config.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, chatConfig.CustomHeaders) - } else { - // SDK 默认未显式设置时 HTTPClient 为 nil,此时构造一个新的注入了 header 的 client。 - config.HTTPClient = secutils.WrapHTTPClientWithHeaders(nil, chatConfig.CustomHeaders) - } + sdkHTTPClient = secutils.WrapHTTPClientWithHeaders(sdkHTTPClient, chatConfig.CustomHeaders) } + config.HTTPClient = sdkHTTPClient modelName := chatConfig.ModelName if chatConfig.ExtraConfig != nil { diff --git a/internal/models/chat/transport.go b/internal/models/chat/transport.go index 8496bd1f..91b3dc26 100644 --- a/internal/models/chat/transport.go +++ b/internal/models/chat/transport.go @@ -48,12 +48,15 @@ func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, cont // Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout) // rather than http.Client.Timeout, so streaming calls are not prematurely terminated. // Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer. -var rawHTTPClient = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: secutils.SSRFSafeDialContext, - TLSHandshakeTimeout: 10 * time.Second, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, +var rawHTTPTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: secutils.SSRFSafeDialContext, + TLSHandshakeTimeout: 10 * time.Second, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 5, } + +var rawHTTPClient = secutils.NewSSRFSafeHTTPClientWithTransport( + secutils.SSRFSafeHTTPClientConfig{Timeout: 0, MaxRedirects: 10}, + rawHTTPTransport, +) diff --git a/internal/models/embedding/transport_test.go b/internal/models/embedding/transport_test.go index 551406a6..5ea3358a 100644 --- a/internal/models/embedding/transport_test.go +++ b/internal/models/embedding/transport_test.go @@ -1,9 +1,12 @@ package embedding import ( + "net/http" "strings" "testing" "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" ) func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) { @@ -15,10 +18,18 @@ func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) { if first == second { t.Fatal("expected distinct HTTP clients") } - if first.Transport != second.Transport { - t.Fatal("expected embedding HTTP clients to share a transport") + firstGuard, ok := first.Transport.(*secutils.SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating transport, got %T", first.Transport) } - if first.Transport != sharedEmbeddingHTTPTransport { + secondGuard, ok := second.Transport.(*secutils.SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating transport, got %T", second.Transport) + } + if firstGuard.Base != secondGuard.Base { + t.Fatal("expected embedding HTTP clients to share a base transport") + } + if firstGuard.Base != http.RoundTripper(sharedEmbeddingHTTPTransport) { t.Fatal("expected embedding HTTP client to use the shared transport") } if first.Timeout != firstTimeout { diff --git a/internal/utils/extraheaders.go b/internal/utils/extraheaders.go index fd504466..b32d54f8 100644 --- a/internal/utils/extraheaders.go +++ b/internal/utils/extraheaders.go @@ -75,7 +75,7 @@ func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) * return client } if client == nil { - client = &http.Client{} + client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig()) } base := client.Transport if base == nil { diff --git a/internal/utils/extraheaders_test.go b/internal/utils/extraheaders_test.go index e24f972a..5f4ecf33 100644 --- a/internal/utils/extraheaders_test.go +++ b/internal/utils/extraheaders_test.go @@ -50,6 +50,9 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) { } func TestWrapHTTPClientWithHeaders(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost") + ResetSSRFWhitelistForTest() + t.Cleanup(ResetSSRFWhitelistForTest) gotTrace := "" gotAuth := "" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/utils/httputil.go b/internal/utils/httputil.go index b1d3ed0a..9fd05cca 100644 --- a/internal/utils/httputil.go +++ b/internal/utils/httputil.go @@ -8,7 +8,10 @@ import ( "time" ) -var defaultHTTPClient = &http.Client{Timeout: 60 * time.Second} +var defaultHTTPClient = NewSSRFSafeHTTPClient(SSRFSafeHTTPClientConfig{ + Timeout: 60 * time.Second, + MaxRedirects: 10, +}) // DownloadBytes fetches the content at the given HTTP(S) URL and returns the // raw bytes. It reuses a package-level http.Client with a 60-second timeout. @@ -16,6 +19,9 @@ func DownloadBytes(url string) ([]byte, error) { if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { return nil, fmt.Errorf("unsupported URL scheme: %s", url) } + if err := ValidateURLForSSRF(url); err != nil { + return nil, fmt.Errorf("URL rejected by SSRF policy: %w", err) + } resp, err := defaultHTTPClient.Get(url) if err != nil { return nil, fmt.Errorf("HTTP GET: %w", err) diff --git a/internal/utils/mysql_ssrf.go b/internal/utils/mysql_ssrf.go new file mode 100644 index 00000000..e45dfc90 --- /dev/null +++ b/internal/utils/mysql_ssrf.go @@ -0,0 +1,27 @@ +package utils + +import ( + "context" + "net" + "sync" + + "github.com/go-sql-driver/mysql" +) + +// MySQLSSRFNetwork is the mysql driver network name registered by +// RegisterMySQLSSRFDialer. DSNs must set cfg.Net to this value so every +// database/sql open uses SSRFSafeDialContext at the final TCP sink. +const MySQLSSRFNetwork = "tcp-weknora-ssrf" + +var mysqlSSRFDialerOnce sync.Once + +// RegisterMySQLSSRFDialer installs a go-sql-driver/mysql dialer that routes +// connections through SSRFSafeDialContext. Safe to call from multiple packages; +// registration happens at most once per process. +func RegisterMySQLSSRFDialer() { + mysqlSSRFDialerOnce.Do(func() { + mysql.RegisterDialContext(MySQLSSRFNetwork, func(ctx context.Context, addr string) (net.Conn, error) { + return SSRFSafeDialContext(ctx, "tcp", addr) + }) + }) +} diff --git a/internal/utils/security.go b/internal/utils/security.go index 5b8b4157..df642778 100644 --- a/internal/utils/security.go +++ b/internal/utils/security.go @@ -254,6 +254,26 @@ var restrictedIPv4Ranges = []*net.IPNet{ mustParseCIDR("172.20.0.0/16"), } +// restrictedPorts contains non-HTTP service ports that user-controlled URLs +// must not reach. It is checked both during URL validation and again at dial +// time so dynamically discovered URLs cannot bypass the input boundary. +var restrictedPorts = map[string]bool{ + "22": true, // SSH + "23": true, // Telnet + "25": true, // SMTP + "445": true, // SMB + "3389": true, // RDP + "5432": true, // PostgreSQL + "3306": true, // MySQL + "6379": true, // Redis + "27017": true, // MongoDB + "9200": true, // Elasticsearch + "2379": true, // etcd + "2380": true, // etcd + "8500": true, // Consul + "4001": true, // etcd (old) +} + // mustParseCIDR parses a CIDR string and panics on error func mustParseCIDR(s string) *net.IPNet { _, ipNet, err := net.ParseCIDR(s) @@ -459,27 +479,8 @@ func isSSRFSafeURL(rawURL string) (bool, string) { // Check for suspicious port numbers port := parsed.Port() - if port != "" { - // Block common internal service ports - blockedPorts := map[string]bool{ - "22": true, // SSH - "23": true, // Telnet - "25": true, // SMTP - "445": true, // SMB - "3389": true, // RDP - "5432": true, // PostgreSQL - "3306": true, // MySQL - "6379": true, // Redis - "27017": true, // MongoDB - "9200": true, // Elasticsearch - "2379": true, // etcd - "2380": true, // etcd - "8500": true, // Consul - "4001": true, // etcd (old) - } - if blockedPorts[port] { - return false, fmt.Sprintf("port %s is blocked for security reasons", port) - } + if restrictedPorts[port] { + return false, fmt.Sprintf("port %s is blocked for security reasons", port) } return true, "" @@ -818,15 +819,36 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request) if redirectHost != "" && IsSSRFWhitelisted(redirectHost) { return nil } - redirectURL := req.URL.String() - if safe, reason := isSSRFSafeURL(redirectURL); !safe { - return fmt.Errorf("%w: %s", ErrSSRFRedirectBlocked, reason) + if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil { + return fmt.Errorf("%w: %w", ErrSSRFRedirectBlocked, err) } return nil } } +// SSRFValidatingRoundTripper enforces the URL policy for every outbound +// request, including URLs discovered at runtime by SDKs (for example OAuth +// metadata) that never passed through an application handler. Dial-time checks +// remain necessary to pin DNS answers and cover transports that cannot accept +// this wrapper directly. +type SSRFValidatingRoundTripper struct { + Base http.RoundTripper +} + +func (t *SSRFValidatingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req == nil || req.URL == nil { + return nil, fmt.Errorf("outbound request blocked: request URL is required") + } + if t == nil || t.Base == nil { + return nil, fmt.Errorf("outbound request blocked: base transport is required") + } + if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil { + return nil, fmt.Errorf("outbound request blocked by SSRF policy: %w", err) + } + return t.Base.RoundTrip(req) +} + // NewSSRFSafeHTTPClientWithTransport wraps a caller-supplied transport in an // *http.Client carrying the given timeout and the SSRF-aware redirect policy. // Pass a transport from NewSSRFSafeTransport (optionally shared across clients) @@ -834,9 +856,12 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request) func NewSSRFSafeHTTPClientWithTransport( config SSRFSafeHTTPClientConfig, transport http.RoundTripper, ) *http.Client { + if transport == nil { + transport = NewSSRFSafeTransport(config) + } return &http.Client{ Timeout: config.Timeout, - Transport: transport, + Transport: &SSRFValidatingRoundTripper{Base: transport}, CheckRedirect: newSSRFCheckRedirect(config.MaxRedirects), } } @@ -849,12 +874,18 @@ func NewSSRFSafeHTTPClient(config SSRFSafeHTTPClientConfig) *http.Client { return NewSSRFSafeHTTPClientWithTransport(config, NewSSRFSafeTransport(config)) } +// SSRFSafeGRPCDialer is compatible with grpc.WithContextDialer and pins DNS +// answers the same way as SSRFSafeDialContext. +func SSRFSafeGRPCDialer(ctx context.Context, addr string) (net.Conn, error) { + return SSRFSafeDialContext(ctx, "tcp", addr) +} + // SSRFSafeDialContext is a custom dial function that validates the resolved IP addresses // before establishing a connection. This provides an additional layer of SSRF protection // against DNS rebinding attacks during the connection phase. func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, error) { // Parse host and port - host, _, err := net.SplitHostPort(addr) + host, port, err := net.SplitHostPort(addr) if err != nil { return nil, fmt.Errorf("invalid address %s: %w", addr, err) } @@ -870,6 +901,9 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } return dialer.DialContext(ctx, network, addr) } + if restrictedPorts[port] { + return nil, fmt.Errorf("connection blocked: port %s is restricted", port) + } // Check if the host is a restricted hostname hostLower := strings.ToLower(host) @@ -884,11 +918,17 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } } - // Resolve the hostname to IP addresses + // Resolve the hostname once, validate every answer, and then dial one of + // those exact IPs. Dialing the original hostname here would make the + // standard dialer resolve it a second time, leaving a DNS-rebinding window + // between validation and connection establishment. ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, fmt.Errorf("DNS resolution failed for %s: %w", host, err) } + if len(ips) == 0 { + return nil, fmt.Errorf("DNS resolution returned no addresses for %s", host) + } // Validate all resolved IPs for _, ipAddr := range ips { @@ -897,13 +937,22 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } } - // If we get here, all IPs are safe. Connect using the standard dialer. - // We dial the original address so that proper connection routing happens. + // If we get here, all IPs are safe. Pin the connection to the validated DNS + // answers; TLS still uses the request hostname for SNI/certificate checks. dialer := &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, } - return dialer.DialContext(ctx, network, addr) + var lastErr error + for _, ipAddr := range ips { + pinnedAddr := net.JoinHostPort(ipAddr.IP.String(), port) + conn, dialErr := dialer.DialContext(ctx, network, pinnedAddr) + if dialErr == nil { + return conn, nil + } + lastErr = dialErr + } + return nil, fmt.Errorf("failed to connect to validated addresses for %s: %w", host, lastErr) } // --------------------------------------------------------------------------- @@ -987,6 +1036,7 @@ func loadSSRFWhitelist() *ssrfWhitelistConfig { // applySSRFWhitelist for the canonical merge logic. func SetSSRFWhitelistFromRaw(raw string) { ssrfWhitelistAtomic.Store(parseSSRFWhitelistRaw(raw)) + invalidateSSRFOutboundValidationCache() } // parseSSRFWhitelistRaw parses a comma-separated whitelist string into @@ -1165,6 +1215,7 @@ func ResetSSRFWhitelistForTest() { ssrfWhitelistOnce = sync.Once{} ssrfWhitelist = nil ssrfWhitelistAtomic.Store(nil) + invalidateSSRFOutboundValidationCache() } // FormatSSRFError takes the error returned by ValidateURLForSSRF and wraps @@ -1242,6 +1293,13 @@ func ValidateURLForSSRF(rawURL string) error { return fmt.Errorf("URL has no hostname") } + // A whitelist relaxes host/IP restrictions only. It must never turn other + // schemes (file://, gopher://, etc.) into valid outbound request targets. + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("invalid scheme: %s (only http/https allowed)", scheme) + } + // If the host is whitelisted, skip the heavy checks. if IsSSRFWhitelisted(hostname) { return nil diff --git a/internal/utils/security_test.go b/internal/utils/security_test.go index 22fba126..30e55628 100644 --- a/internal/utils/security_test.go +++ b/internal/utils/security_test.go @@ -191,6 +191,12 @@ func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) { rawURL: "https://8.8.8.8/dns-query", wantErr: true, }, + { + name: "whitelist does not allow non-http scheme", + whitelist: "example.com", + rawURL: "gopher://example.com/_stats", + wantErr: true, + }, } for _, tt := range tests { diff --git a/internal/utils/security_transport_test.go b/internal/utils/security_transport_test.go index 296e2417..303df5ae 100644 --- a/internal/utils/security_transport_test.go +++ b/internal/utils/security_transport_test.go @@ -1,11 +1,46 @@ package utils import ( + "context" + "fmt" "net/http" + "strings" "testing" "time" ) +type recordingRoundTripper struct { + called bool +} + +func (r *recordingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + r.called = true + return nil, fmt.Errorf("unexpected network call") +} + +func TestSSRFSafeClientValidatesInitialRequestAtFinalSink(t *testing.T) { + base := &recordingRoundTripper{} + client := NewSSRFSafeHTTPClientWithTransport(DefaultSSRFSafeHTTPClientConfig(), base) + req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data", nil) + if err != nil { + t.Fatal(err) + } + _, err = client.Do(req) + if err == nil || !strings.Contains(err.Error(), "SSRF") { + t.Fatalf("expected final-sink SSRF rejection, got %v", err) + } + if base.called { + t.Fatal("unsafe request reached the base transport") + } +} + +func TestSSRFSafeDialContextRejectsRestrictedPortAtFinalSink(t *testing.T) { + _, err := SSRFSafeDialContext(context.Background(), "tcp", "example.com:6379") + if err == nil || !strings.Contains(err.Error(), "port 6379") { + t.Fatalf("expected restricted-port error, got %v", err) + } +} + // TestNewSSRFSafeTransport_SharedAcrossClients verifies that a single transport // can back multiple clients (global connection pooling) while each client keeps // its own timeout and a redirect policy. @@ -22,11 +57,16 @@ func TestNewSSRFSafeTransport_SharedAcrossClients(t *testing.T) { if first == second { t.Fatal("expected distinct HTTP clients") } - if first.Transport != second.Transport { - t.Fatal("expected clients to share the same transport") + firstGuard, ok := first.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", first.Transport) } - if first.Transport != http.RoundTripper(shared) { - t.Fatal("expected client to use the supplied shared transport") + secondGuard, ok := second.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", second.Transport) + } + if firstGuard.Base != secondGuard.Base || firstGuard.Base != http.RoundTripper(shared) { + t.Fatal("expected clients to share the supplied base transport") } if first.Timeout != 15*time.Second { t.Fatalf("unexpected first timeout: got %v, want %v", first.Timeout, 15*time.Second) @@ -46,8 +86,12 @@ func TestNewSSRFSafeHTTPClient_HasDedicatedTransport(t *testing.T) { if client.Transport == nil { t.Fatal("expected a transport to be set") } - if _, ok := client.Transport.(*http.Transport); !ok { - t.Fatalf("expected *http.Transport, got %T", client.Transport) + guard, ok := client.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", client.Transport) + } + if _, ok := guard.Base.(*http.Transport); !ok { + t.Fatalf("expected *http.Transport base, got %T", guard.Base) } if client.CheckRedirect == nil { t.Fatal("expected SSRF redirect policy to be set") diff --git a/internal/utils/ssrf_outbound_cache.go b/internal/utils/ssrf_outbound_cache.go new file mode 100644 index 00000000..d7ffdbba --- /dev/null +++ b/internal/utils/ssrf_outbound_cache.go @@ -0,0 +1,135 @@ +package utils + +import ( + "fmt" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/singleflight" +) + +// outboundSSRFValidationTTL bounds how long a successful RoundTrip/redirect +// validation for the same origin may be reused. DNS rebinding at the TCP sink +// is still blocked by SSRFSafeDialContext on every new connection. +const outboundSSRFValidationTTL = 60 * time.Second + +var ( + ssrfOutboundCacheGen atomic.Uint64 + ssrfOutboundCache sync.Map // string -> *ssrfOutboundCacheEntry + ssrfOutboundValidateGroup singleflight.Group + ssrfOutboundValidateMisses atomic.Uint64 // test-only counter +) + +type ssrfOutboundCacheEntry struct { + err error + expiresAt time.Time +} + +// validateURLForSSRFForOutbound validates URLs on the hot outbound path +// (RoundTripper, redirect checks). Results are cached per origin for a short +// TTL so high-frequency clients do not repeat DNS lookups on every request. +// Handler/input boundaries should keep calling ValidateURLForSSRF directly. +func validateURLForSSRFForOutbound(rawURL string) error { + if rawURL == "" { + return nil + } + + cacheKey, ok := outboundSSRFCacheKey(rawURL) + if !ok { + return ValidateURLForSSRF(rawURL) + } + + now := time.Now() + if cached, ok := ssrfOutboundCache.Load(cacheKey); ok { + entry := cached.(*ssrfOutboundCacheEntry) + if now.Before(entry.expiresAt) { + return entry.err + } + ssrfOutboundCache.Delete(cacheKey) + } + + result, err, _ := ssrfOutboundValidateGroup.Do(cacheKey, func() (any, error) { + if cached, ok := ssrfOutboundCache.Load(cacheKey); ok { + entry := cached.(*ssrfOutboundCacheEntry) + if time.Now().Before(entry.expiresAt) { + return entry.err, entry.err + } + } + + ssrfOutboundValidateMisses.Add(1) + validationErr := ValidateURLForSSRF(rawURL) + ssrfOutboundCache.Store(cacheKey, &ssrfOutboundCacheEntry{ + err: validationErr, + expiresAt: time.Now().Add(outboundSSRFValidationTTL), + }) + return validationErr, validationErr + }) + if err != nil { + return err + } + if validationErr, ok := result.(error); ok { + return validationErr + } + return nil +} + +func outboundSSRFCacheKey(rawURL string) (string, bool) { + normalized := rawURL + if !strings.Contains(normalized, "://") { + normalized = "https://" + normalized + } + parsed, err := url.Parse(normalized) + if err != nil { + return "", false + } + + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "", false + } + + host := strings.ToLower(parsed.Hostname()) + if host == "" { + return "", false + } + + port := parsed.Port() + if port == "" { + if scheme == "https" { + port = "443" + } else { + port = "80" + } + } + + var origin string + if strings.Contains(host, ":") { + origin = fmt.Sprintf("%s://[%s]:%s", scheme, host, port) + } else { + origin = fmt.Sprintf("%s://%s:%s", scheme, host, port) + } + + return fmt.Sprintf("%d|%s", ssrfOutboundCacheGen.Load(), origin), true +} + +func invalidateSSRFOutboundValidationCache() { + ssrfOutboundCacheGen.Add(1) + ssrfOutboundCache = sync.Map{} + ssrfOutboundValidateGroup = singleflight.Group{} +} + +// ResetSSRFOutboundValidationCacheForTest clears the outbound validation cache. +// NOT for production use. +func ResetSSRFOutboundValidationCacheForTest() { + invalidateSSRFOutboundValidationCache() + ssrfOutboundValidateMisses.Store(0) +} + +// SSRFOutboundValidationMissesForTest returns how many uncached outbound +// validations have run since the last ResetSSRFOutboundValidationCacheForTest. +func SSRFOutboundValidationMissesForTest() uint64 { + return ssrfOutboundValidateMisses.Load() +} diff --git a/internal/utils/ssrf_outbound_cache_test.go b/internal/utils/ssrf_outbound_cache_test.go new file mode 100644 index 00000000..b3856478 --- /dev/null +++ b/internal/utils/ssrf_outbound_cache_test.go @@ -0,0 +1,101 @@ +package utils + +import ( + "net/http" + "testing" + "time" +) + +func TestOutboundSSRFValidationCacheReusesSameOrigin(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/a"); err != nil { + t.Fatalf("first validation failed: %v", err) + } + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/b"); err != nil { + t.Fatalf("second validation failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 1 { + t.Fatalf("expected 1 cache miss for same origin, got %d", got) + } +} + +func TestOutboundSSRFValidationCacheInvalidatesOnWhitelistChange(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil { + t.Fatalf("initial validation failed: %v", err) + } + if SSRFOutboundValidationMissesForTest() != 1 { + t.Fatalf("expected one miss before whitelist change, got %d", SSRFOutboundValidationMissesForTest()) + } + + SetSSRFWhitelistFromRaw("cache-test.example.com,other.example.com") + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil { + t.Fatalf("validation after whitelist change failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 2 { + t.Fatalf("expected cache miss after whitelist bump, got %d misses", got) + } +} + +func TestOutboundSSRFValidationCacheExpires(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil { + t.Fatalf("initial validation failed: %v", err) + } + + cacheKey, ok := outboundSSRFCacheKey("https://cache-test.example.com/a") + if !ok { + t.Fatal("expected cache key") + } + raw, ok := ssrfOutboundCache.Load(cacheKey) + if !ok { + t.Fatal("expected cached entry") + } + entry := raw.(*ssrfOutboundCacheEntry) + entry.expiresAt = time.Now().Add(-time.Second) + ssrfOutboundCache.Store(cacheKey, entry) + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil { + t.Fatalf("validation after expiry failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 2 { + t.Fatalf("expected revalidation after TTL expiry, got %d misses", got) + } +} + +func TestSSRFValidatingRoundTripperUsesOutboundCache(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + base := &recordingRoundTripper{} + rt := &SSRFValidatingRoundTripper{Base: base} + + req1, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/a", nil) + if err != nil { + t.Fatal(err) + } + req2, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/b", nil) + if err != nil { + t.Fatal(err) + } + + if _, err := rt.RoundTrip(req1); err == nil { + t.Fatal("expected base transport error, validation should pass") + } + if _, err := rt.RoundTrip(req2); err == nil { + t.Fatal("expected base transport error, validation should pass") + } + if got := SSRFOutboundValidationMissesForTest(); got != 1 { + t.Fatalf("round tripper expected one outbound validation miss, got %d", got) + } +}