From 1030832c4ba22a407dfde5d6dffbe0bb62caae20 Mon Sep 17 00:00:00 2001 From: lyingbug Date: Mon, 10 Aug 2026 13:24:14 +0800 Subject: [PATCH] fix(security): harden outbound URL SSRF protection (#2626) * fix(security): harden outbound URL SSRF protection * test(security): add SSRF validation tests for outbound URL caching Introduce tests for outbound SSRF validation cache to ensure proper handling of URL validation, including cache reuse for the same origin, cache invalidation on whitelist changes, and expiration of cached entries. This enhances the robustness of SSRF protection mechanisms in the application. * refactor: clean up whitespace in engine_factory.go and ssrf_outbound_cache.go Removed unnecessary blank lines in engine_factory.go and adjusted variable declarations for consistency in ssrf_outbound_cache.go. This improves code readability without altering functionality. --- .env.example | 4 +- docker-compose.yml | 10 +- docreader/parser/opendataloader_parser.py | 20 ++- docreader/tests/test_opendataloader_parser.py | 30 ++++ docreader/tests/test_ssrf.py | 15 ++ docreader/utils/ssrf.py | 20 ++- .../retriever/opensearch/ssrf_test.go | 16 +++ .../retriever/opensearch/transport.go | 8 +- internal/application/service/file/cos.go | 30 ++-- .../service/file/endpoint_security_test.go | 41 ++++++ internal/application/service/file/ks3.go | 4 + internal/application/service/file/minio.go | 7 + internal/application/service/file/obs.go | 9 ++ internal/application/service/file/oss.go | 6 +- internal/application/service/file/oss_test.go | 8 +- internal/application/service/file/s3.go | 9 +- internal/application/service/file/tos.go | 16 +++ .../application/service/knowledge_process.go | 10 ++ .../service/knowledge_util_ssrf_test.go | 4 +- internal/application/service/mcp_service.go | 6 + .../service/parser_url_security.go | 33 +++++ .../service/parser_url_security_test.go | 24 ++++ .../service/vectorstore_healthcheck.go | 27 ++-- .../vectorstore_healthcheck_ssrf_test.go | 46 ++++++ internal/application/service/weknoracloud.go | 4 +- internal/container/engine_factory.go | 84 +++++++++-- internal/container/engine_factory_test.go | 8 +- internal/container/ssrf_test.go | 43 ++++++ internal/handler/mcp_service.go | 11 ++ internal/handler/system.go | 69 ++------- internal/handler/tenant.go | 10 ++ internal/im/dingtalk/adapter.go | 8 +- internal/im/dingtalk/download_test.go | 11 ++ internal/im/feishu/adapter.go | 5 +- internal/im/feishu/send_e2e_test.go | 11 ++ internal/im/qqbot/longconn.go | 5 +- internal/im/telegram/adapter.go | 12 +- internal/im/telegram/longconn.go | 10 +- internal/im/wechat/adapter.go | 9 +- internal/im/wechat/longpoll.go | 46 +++--- internal/im/wechat/qrcode.go | 14 +- internal/im/wecom/longconn.go | 5 +- internal/im/wecom/webhook_adapter.go | 5 +- .../infrastructure/docparser/http_parser.go | 34 +++-- .../docparser/paddleocr_vl_cloud_converter.go | 3 + .../docparser/paddleocr_vl_converter.go | 11 +- .../docparser/weknoracloud_http_reader.go | 12 +- internal/infrastructure/web_search/baidu.go | 5 +- internal/infrastructure/web_search/ollama.go | 5 +- internal/infrastructure/web_search/proxy.go | 19 +-- .../infrastructure/web_search/proxy_test.go | 7 +- internal/mcp/client.go | 14 +- internal/mcp/oauth_manager.go | 7 + internal/mcp/security.go | 36 +++++ internal/mcp/security_test.go | 31 ++++ internal/models/chat/remote_api.go | 12 +- internal/models/chat/transport.go | 19 +-- internal/models/embedding/transport_test.go | 17 ++- internal/utils/extraheaders.go | 2 +- internal/utils/extraheaders_test.go | 3 + internal/utils/httputil.go | 8 +- internal/utils/mysql_ssrf.go | 27 ++++ internal/utils/security.go | 118 +++++++++++---- internal/utils/security_test.go | 6 + internal/utils/security_transport_test.go | 56 +++++++- internal/utils/ssrf_outbound_cache.go | 135 ++++++++++++++++++ internal/utils/ssrf_outbound_cache_test.go | 101 +++++++++++++ 67 files changed, 1214 insertions(+), 247 deletions(-) create mode 100644 internal/application/repository/retriever/opensearch/ssrf_test.go create mode 100644 internal/application/service/file/endpoint_security_test.go create mode 100644 internal/application/service/parser_url_security.go create mode 100644 internal/application/service/parser_url_security_test.go create mode 100644 internal/application/service/vectorstore_healthcheck_ssrf_test.go create mode 100644 internal/container/ssrf_test.go create mode 100644 internal/mcp/security.go create mode 100644 internal/mcp/security_test.go create mode 100644 internal/utils/mysql_ssrf.go create mode 100644 internal/utils/ssrf_outbound_cache.go create mode 100644 internal/utils/ssrf_outbound_cache_test.go diff --git a/.env.example b/.env.example index c1e874d7..b9876e98 100644 --- a/.env.example +++ b/.env.example @@ -717,10 +717,10 @@ LANGFUSE_HOST=http://langfuse-web:3000 # # ===== 向量库地址的 SSRF 校验 ===== # 通过 API 创建/测试向量库连接时,地址会经过 SSRF 校验。docker-compose 内置服务 -# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。 +# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng, minio)已通过 SSRF_WHITELIST_EXTRA 默认放行。 # 如使用外部私网地址或被拦截端口(5432, 9200 等),请将主机加入 SSRF_WHITELIST_EXTRA # (注意:自定义该变量会覆盖 compose 默认值,需重新包含内置服务名),例如: -# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,my-opensearch.internal +# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio,my-opensearch.internal # # Doris Stream Load:FE 会将请求 307 重定向到 BE;docker compose 默认放行 # doris-fe 与 doris-be(见 SSRF_WHITELIST_EXTRA)。 diff --git a/docker-compose.yml b/docker-compose.yml index 51550382..1ecb490b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -285,17 +285,17 @@ services: - SSRF_WHITELIST=${SSRF_WHITELIST:-} # 保留原始 URL 的图片域名白名单(逗号分隔,不替换为 provider://) - IMAGE_HOST_KEEP_URL=${IMAGE_HOST_KEEP_URL:-} - # Always allow the optional searxng sidecar and the bundled vector-store - # service hostnames (compose service names). Vector-store create/test now - # runs SSRF validation on user-supplied addresses, which would otherwise - # reject these private docker-network hosts; whitelisting them keeps the + # Always allow the optional searxng sidecar, bundled vector stores, and + # the default MinIO storage host (compose service names). User-controlled + # endpoints are SSRF-validated and these private docker-network hosts + # would otherwise be rejected; whitelisting them keeps the # out-of-box experience. Merged on top of SSRF_WHITELIST so user # overrides don't clobber it. (postgres is intentionally NOT listed: it # is not a user-registerable vector store, and whitelisting the app's own # DB host would skip the 5432 port block.) # doris-be: Stream Load follows FE→BE redirects that must receive Basic # credentials on whitelisted BE hosts. - - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be} + - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} # File size limit (in MB) diff --git a/docreader/parser/opendataloader_parser.py b/docreader/parser/opendataloader_parser.py index f6374c4f..bdb96c28 100644 --- a/docreader/parser/opendataloader_parser.py +++ b/docreader/parser/opendataloader_parser.py @@ -25,6 +25,7 @@ from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.base_parser import BaseParser from docreader.parser.concurrency import parser_worker_limit +from docreader.utils.ssrf import is_ssrf_safe_url logger = logging.getLogger(__name__) @@ -34,6 +35,13 @@ _MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") _IMAGE_FILE_NUM_RE = re.compile(r"^imageFile(\d+)\.", re.I) +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Health probes never need redirects; refusing them closes redirect SSRF.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + def _override_str(overrides: Optional[Mapping[str, Any]], key: str, default: str = "") -> str: if overrides: v = overrides.get(key) @@ -75,11 +83,16 @@ def _ping_hybrid( base = url.rstrip("/") health_url = f"{base}/health" + safe, reason = is_ssrf_safe_url(health_url) + if not safe: + return False, f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}" + + opener = urllib.request.build_opener(_NoRedirectHandler()) last_err = "" for attempt in range(max(1, retries)): try: req = urllib.request.Request(health_url, method="GET") - with urllib.request.urlopen(req, timeout=timeout_sec) as resp: + with opener.open(req, timeout=timeout_sec) as resp: if 200 <= resp.status < 300: return True, "" last_err = f"hybrid 健康检查 HTTP {resp.status}: {health_url}" @@ -278,6 +291,11 @@ def _run_convert( kwargs["hybrid"] = hybrid hybrid_url = _resolve_hybrid_url(overrides) if hybrid_url: + safe, reason = is_ssrf_safe_url(hybrid_url) + if not safe: + raise RuntimeError( + f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}" + ) kwargs["hybrid_url"] = hybrid_url hybrid_mode = _override_str(overrides, "odl_hybrid_mode", CONFIG.odl_hybrid_mode) if hybrid_mode: diff --git a/docreader/tests/test_opendataloader_parser.py b/docreader/tests/test_opendataloader_parser.py index 28a6f974..70792dcd 100644 --- a/docreader/tests/test_opendataloader_parser.py +++ b/docreader/tests/test_opendataloader_parser.py @@ -1,6 +1,7 @@ """Unit tests for OpenDataLoader parser helpers (no JVM required).""" import os +import sys import tempfile import unittest from unittest import mock @@ -10,12 +11,41 @@ from docreader.parser.opendataloader_parser import ( _collect_images_under_output, _find_markdown_file, _normalize_odl_image_url, + _ping_hybrid, + _run_convert, _rewrite_markdown_image_refs, opendataloader_available, ) class OpenDataLoaderHelpersTest(unittest.TestCase): + def test_hybrid_health_probe_blocks_private_url_before_request(self): + with mock.patch( + "docreader.parser.opendataloader_parser.is_ssrf_safe_url", + return_value=(False, "restricted test address"), + ), mock.patch("urllib.request.build_opener") as build_opener: + ok, msg = _ping_hybrid("http://127.0.0.1:8080", retries=1) + + self.assertFalse(ok) + self.assertIn("SSRF", msg) + build_opener.assert_not_called() + + def test_convert_blocks_private_hybrid_url_at_final_sink(self): + fake_module = mock.Mock() + with mock.patch.dict(sys.modules, {"opendataloader_pdf": fake_module}): + with self.assertRaisesRegex(RuntimeError, "SSRF"): + _run_convert( + "/tmp/input.pdf", + "/tmp/output", + "/tmp/output/images", + overrides={ + "odl_hybrid": "docling-fast", + "odl_hybrid_url": "http://169.254.169.254/latest/meta-data", + }, + ) + + fake_module.convert.assert_not_called() + def test_find_markdown_prefers_stem_match(self): with tempfile.TemporaryDirectory() as d: other = os.path.join(d, "other.md") diff --git a/docreader/tests/test_ssrf.py b/docreader/tests/test_ssrf.py index d42b6468..a4f944db 100644 --- a/docreader/tests/test_ssrf.py +++ b/docreader/tests/test_ssrf.py @@ -1,3 +1,4 @@ +import ipaddress import os import unittest from unittest.mock import patch @@ -36,6 +37,20 @@ class TestSSRFValidation(unittest.TestCase): self.assertFalse(safe) self.assertTrue(reason) + def test_blocks_invalid_port(self): + safe, reason = is_ssrf_safe_url("https://example.com:99999/path") + self.assertFalse(safe) + self.assertIn("invalid port", reason) + + def test_blocks_ipv4_mapped_private_ipv6_resolution(self): + with patch( + "docreader.utils.ssrf._resolve_host_ips", + return_value=((ipaddress.ip_address("::ffff:127.0.0.1"),), None), + ): + safe, reason = is_ssrf_safe_url("https://example.invalid/path") + self.assertFalse(safe) + self.assertIn("restricted", reason) + def test_allows_public_https(self): safe, reason = is_ssrf_safe_url("https://example.com/article") self.assertTrue(safe, reason) diff --git a/docreader/utils/ssrf.py b/docreader/utils/ssrf.py index c13f6a41..be33e004 100644 --- a/docreader/utils/ssrf.py +++ b/docreader/utils/ssrf.py @@ -161,6 +161,21 @@ def _is_restricted_ip(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) - if isinstance(net, ipaddress.IPv4Network) and ip in net: return f"restricted range {net}" if isinstance(ip, ipaddress.IPv6Address): + embedded_ipv4 = ip.ipv4_mapped + if embedded_ipv4 is not None: + reason = _is_restricted_ip(embedded_ipv4) + if reason: + return f"IPv4-mapped {reason}" + if ip.sixtofour is not None: + reason = _is_restricted_ip(ip.sixtofour) + if reason: + return f"6to4-embedded {reason}" + if ip.teredo is not None: + server_ip, client_ip = ip.teredo + for label, embedded_ip in (("server", server_ip), ("client", client_ip)): + reason = _is_restricted_ip(embedded_ip) + if reason: + return f"Teredo {label} embeds {reason}" # Site-local (fec0::/10) if (ip.packed[0] == 0xFE) and (ip.packed[1] & 0xC0) == 0xC0: return "site-local IPv6 address" @@ -238,7 +253,10 @@ def is_ssrf_safe_url(raw_url: str) -> Tuple[bool, str]: f"hostname {hostname_lower} resolves to restricted IP {resolved_ip}: {reason}", ) - port = parsed.port + try: + port = parsed.port + except ValueError as exc: + return False, f"invalid port: {exc}" if port is not None and str(port) in BLOCKED_PORTS: return False, f"port {port} is blocked for security reasons" diff --git a/internal/application/repository/retriever/opensearch/ssrf_test.go b/internal/application/repository/retriever/opensearch/ssrf_test.go new file mode 100644 index 00000000..790c9e87 --- /dev/null +++ b/internal/application/repository/retriever/opensearch/ssrf_test.go @@ -0,0 +1,16 @@ +package opensearch + +import ( + "os" + "testing" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + // End-to-end repository tests intentionally use loopback httptest servers. + secutils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost,opensearch.example.com") + code := m.Run() + secutils.SetSSRFWhitelistFromRaw("") + os.Exit(code) +} diff --git a/internal/application/repository/retriever/opensearch/transport.go b/internal/application/repository/retriever/opensearch/transport.go index c39fda27..fbe0e1b6 100644 --- a/internal/application/repository/retriever/opensearch/transport.go +++ b/internal/application/repository/retriever/opensearch/transport.go @@ -10,6 +10,7 @@ import ( osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // NewOpenSearchClient builds a TLS-hardened, pool-tuned *osapi.Client for @@ -35,13 +36,16 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) { if cfg == nil || cfg.Addr == "" { return nil, fmt.Errorf("opensearch: ConnectionConfig.Addr required: %w", ErrConfigInvalid) } + if err := secutils.ValidateURLForSSRF(cfg.Addr); err != nil { + return nil, fmt.Errorf("opensearch: address failed SSRF validation: %w", err) + } transport := buildHTTPTransport(cfg.InsecureSkipVerify) return osapi.NewClient(osapi.Config{ Client: opensearch.Config{ Addresses: []string{cfg.Addr}, Username: cfg.Username, Password: cfg.Password, - Transport: transport, + Transport: &secutils.SSRFValidatingRoundTripper{Base: transport}, }, }) } @@ -54,6 +58,8 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) { // final client fragile across SDK versions). func buildHTTPTransport(insecureSkipVerify bool) *http.Transport { return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: secutils.SSRFSafeDialContext, TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, InsecureSkipVerify: insecureSkipVerify, //nolint:gosec — operator opt-in flag diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go index 6427bd8f..442584f2 100644 --- a/internal/application/service/file/cos.go +++ b/internal/application/service/file/cos.go @@ -32,6 +32,22 @@ type cosFileService struct { const cosScheme = "cos://" +func newCOSHTTPClient(secretID, secretKey string) *http.Client { + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() + client := utils.NewSSRFSafeHTTPClient(httpConfig) + // COS GetCredential expects AuthorizationTransport to remain the outermost + // transport. Put the per-request SSRF guard immediately underneath it while + // retaining the safe client's redirect policy. + client.Transport = &cos.AuthorizationTransport{ + SecretID: secretID, + SecretKey: secretKey, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }, + } + return client +} + // newCosClient creates a bare cosFileService with just the SDK client initialised. // Shared by NewCosFileService* constructors and CheckCosConnectivity. func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) { @@ -41,12 +57,7 @@ func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileServi if err != nil { return nil, fmt.Errorf("failed to parse bucketURL: %w", err) } - client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: secretID, - SecretKey: secretKey, - }, - }) + client := cos.NewClient(&cos.BaseURL{BucketURL: u}, newCOSHTTPClient(secretID, secretKey)) return &cosFileService{client: client, bucketURL: bucketURL, bucketName: bucketName, region: region}, nil } @@ -72,12 +83,7 @@ func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, co if err != nil { return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err) } - svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: secretId, - SecretKey: secretKey, - }, - }) + svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, newCOSHTTPClient(secretId, secretKey)) svc.tempBucketURL = tempBucketURL } diff --git a/internal/application/service/file/endpoint_security_test.go b/internal/application/service/file/endpoint_security_test.go new file mode 100644 index 00000000..55e6d09f --- /dev/null +++ b/internal/application/service/file/endpoint_security_test.go @@ -0,0 +1,41 @@ +package file + +import ( + "context" + "testing" +) + +func TestStorageClientsRejectUnsafeEndpointAtConstruction(t *testing.T) { + const endpoint = "http://169.254.169.254/latest/meta-data" + tests := map[string]func() error{ + "minio": func() error { + _, err := newMinioClient(endpoint, "ak", "sk", "bucket", false) + return err + }, + "s3": func() error { + _, err := newS3Client(endpoint, "ak", "sk", "bucket", "region", "", true) + return err + }, + "obs": func() error { + return CheckObsConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket") + }, + "oss": func() error { + _, err := newOSSClient(endpoint, "region", "ak", "sk") + return err + }, + "tos": func() error { + return CheckTosConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket") + }, + "ks3": func() error { + _, err := newKS3Client(endpoint, "region", "ak", "sk") + return err + }, + } + for name, run := range tests { + t.Run(name, func(t *testing.T) { + if err := run(); err == nil { + t.Fatal("expected unsafe endpoint to be rejected") + } + }) + } +} diff --git a/internal/application/service/file/ks3.go b/internal/application/service/file/ks3.go index f4489915..07b5528c 100644 --- a/internal/application/service/file/ks3.go +++ b/internal/application/service/file/ks3.go @@ -53,6 +53,9 @@ func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathP } func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe KS3 endpoint: %w", err) + } creds := credentials.NewStaticCredentials(accessKey, secretKey, "") client := ks3s3.New(&ks3aws.Config{ Credentials: creds, @@ -62,6 +65,7 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err S3ForcePathStyle: false, // KS3 uses virtual-hosted style SignerVersion: "V2", // KS3 recommends V2 signing MaxRetries: 3, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) return client, nil } diff --git a/internal/application/service/file/minio.go b/internal/application/service/file/minio.go index 7ef51d4f..7ba43235 100644 --- a/internal/application/service/file/minio.go +++ b/internal/application/service/file/minio.go @@ -28,9 +28,16 @@ type minioFileService struct { // Shared by NewMinioFileService (which also ensures the bucket exists) and // CheckMinioConnectivity (read-only probe). func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe MinIO endpoint: %w", err) + } + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() client, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }, }) if err != nil { return nil, fmt.Errorf("failed to initialize MinIO client: %w", err) diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.go index 1b8ffe8e..451d92db 100644 --- a/internal/application/service/file/obs.go +++ b/internal/application/service/file/obs.go @@ -12,6 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -43,12 +44,16 @@ func NewObsFileService( endpoint, region, accessKeyID, secretAccessKey, bucketName string, pathPrefix string, ) (interfaces.FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe OBS endpoint: %w", err) + } client := s3.New(s3.Options{ Region: region, EndpointResolver: &obsEndpointResolver{url: endpoint}, Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""), UsePathStyle: true, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) _, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{ @@ -76,6 +81,9 @@ func NewObsFileService( } func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("unsafe OBS endpoint: %w", err) + } checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -84,6 +92,7 @@ func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secr EndpointResolver: &obsEndpointResolver{url: endpoint}, Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), UsePathStyle: true, + HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), }) _, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{ diff --git a/internal/application/service/file/oss.go b/internal/application/service/file/oss.go index 48974c77..bde515e6 100644 --- a/internal/application/service/file/oss.go +++ b/internal/application/service/file/oss.go @@ -34,12 +34,16 @@ const ossScheme = "oss://" // newOSSClient creates an OSS client using the official Aliyun SDK v2. func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe OSS endpoint: %w", err) + } creds := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") cfg := oss.LoadDefaultConfig(). WithCredentialsProvider(creds). WithRegion(region). - WithEndpoint(endpoint) + WithEndpoint(endpoint). + WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())) return oss.NewClient(cfg), nil } diff --git a/internal/application/service/file/oss_test.go b/internal/application/service/file/oss_test.go index 4144f29e..679f9402 100644 --- a/internal/application/service/file/oss_test.go +++ b/internal/application/service/file/oss_test.go @@ -117,7 +117,7 @@ func TestNewOSSClient(t *testing.T) { }, { name: "custom endpoint", - endpoint: "https://custom-oss-endpoint.com", + endpoint: "https://example.com", region: "cn-shanghai", accessKey: "ak", secretKey: "sk", @@ -145,6 +145,12 @@ func TestNewOSSClient(t *testing.T) { } } +func TestNewOSSClientRejectsUnsafeEndpoint(t *testing.T) { + if _, err := newOSSClient("http://127.0.0.1:9000", "cn-hangzhou", "ak", "sk"); err == nil { + t.Fatal("expected loopback OSS endpoint to be rejected") + } +} + func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go index ffd40c9d..02da6a02 100644 --- a/internal/application/service/file/s3.go +++ b/internal/application/service/file/s3.go @@ -31,6 +31,9 @@ type s3FileService struct { // newS3Client creates a bare s3FileService with just the SDK client initialised. func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe S3 endpoint: %w", err) + } var cfg aws.Config var err error @@ -55,16 +58,20 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix // Create S3 client with custom endpoint if provided. // For S3-compatible services (non-AWS), use path-style addressing // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key). + httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()) var client *s3.Client if endpoint != "" { usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com") client = s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpoint) o.UsePathStyle = usePathStyle + o.HTTPClient = httpClient }) } else { // Standard AWS S3 - client = s3.NewFromConfig(cfg) + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.HTTPClient = httpClient + }) } // Normalize pathPrefix: ensure it ends with '/' if not empty diff --git a/internal/application/service/file/tos.go b/internal/application/service/file/tos.go index 39f5f344..a260f7f3 100644 --- a/internal/application/service/file/tos.go +++ b/internal/application/service/file/tos.go @@ -36,10 +36,17 @@ func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathP // NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket. func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("unsafe TOS endpoint: %w", err) + } + httpConfig := utils.DefaultSSRFSafeHTTPClientConfig() client, err := tos.NewClientV2( endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }), ) if err != nil { return nil, fmt.Errorf("failed to initialize TOS client: %w", err) @@ -58,6 +65,9 @@ func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, buc endpoint, tos.WithRegion(tempRegion), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(httpConfig), + }), ) if err != nil { return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err) @@ -87,10 +97,16 @@ func (s *tosFileService) CheckConnectivity(ctx context.Context) error { // CheckTosConnectivity tests TOS connectivity using the provided credentials. func CheckTosConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("unsafe TOS endpoint: %w", err) + } client, err := tos.NewClientV2( endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }), ) if err != nil { return fmt.Errorf("failed to initialize TOS client: %w", err) diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 64b9eb68..fb539633 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3593,6 +3593,16 @@ func (s *knowledgeService) convert( } mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides) applyParserRuleOverrides(mergedOverrides, eff.ChunkingConfig, fileType) + if err := validateParserEngineOverrideURLs(mergedOverrides); err != nil { + logger.Errorf(ctx, "Parser endpoint rejected for SSRF protection: %v", err) + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "Parser endpoint is not allowed for security reasons" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + s.failStage(ctx, knowledge.ID, types.StageDocReader, + werrors.ErrCodeDocReaderParseFailed, knowledge.ErrorMessage, err) + return nil, nil + } if isURL { if err := secutils.ValidateURLForSSRF(payload.URL); err != nil { diff --git a/internal/application/service/knowledge_util_ssrf_test.go b/internal/application/service/knowledge_util_ssrf_test.go index c3e8d0a9..6a2a4df1 100644 --- a/internal/application/service/knowledge_util_ssrf_test.go +++ b/internal/application/service/knowledge_util_ssrf_test.go @@ -33,7 +33,9 @@ func TestDownloadFileFromURLBlocksRedirectToLoopback(t *testing.T) { if err == nil { t.Fatal("expected redirect to loopback to be blocked") } - if !strings.Contains(err.Error(), "redirect blocked") && !strings.Contains(err.Error(), "connection blocked") { + if !strings.Contains(err.Error(), "redirect blocked") && + !strings.Contains(err.Error(), "connection blocked") && + !strings.Contains(err.Error(), "outbound request blocked") { t.Fatalf("unexpected error: %v", err) } } diff --git a/internal/application/service/mcp_service.go b/internal/application/service/mcp_service.go index 9aadfd74..8899641d 100644 --- a/internal/application/service/mcp_service.go +++ b/internal/application/service/mcp_service.go @@ -41,6 +41,9 @@ func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types if service.TransportType == types.MCPTransportStdio { return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead") } + if err := mcp.ValidateServiceOutboundURLs(service); err != nil { + return err + } // Set default advanced config if not provided if service.AdvancedConfig == nil { @@ -248,6 +251,9 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types } // Update timestamp + if err := mcp.ValidateServiceOutboundURLs(existing); err != nil { + return err + } existing.UpdatedAt = time.Now() if err := s.mcpServiceRepo.Update(ctx, existing); err != nil { diff --git a/internal/application/service/parser_url_security.go b/internal/application/service/parser_url_security.go new file mode 100644 index 00000000..cb2d3db3 --- /dev/null +++ b/internal/application/service/parser_url_security.go @@ -0,0 +1,33 @@ +package service + +import ( + "fmt" + "strings" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +var parserOutboundURLKeys = []string{ + "mineru_endpoint", + "mineru_vlm_server_url", + "odl_hybrid_url", + "paddleocr_vl_endpoint", + "paddleocr_vl_cloud_base_url", +} + +// validateParserEngineOverrideURLs validates every parser override that can +// cause this process or the trusted DocReader service to make an outbound +// request. Per-upload overrides are included because API callers can provide +// the generic parser_engine_overrides map directly. +func validateParserEngineOverrideURLs(overrides map[string]string) error { + for _, key := range parserOutboundURLKeys { + rawURL := strings.TrimSpace(overrides[key]) + if rawURL == "" { + continue + } + if err := secutils.ValidateURLForSSRF(rawURL); err != nil { + return fmt.Errorf("%s failed SSRF validation: %w", key, err) + } + } + return nil +} diff --git a/internal/application/service/parser_url_security_test.go b/internal/application/service/parser_url_security_test.go new file mode 100644 index 00000000..87c1efe8 --- /dev/null +++ b/internal/application/service/parser_url_security_test.go @@ -0,0 +1,24 @@ +package service + +import "testing" + +func TestValidateParserOverrideURLsRejectsEverySupportedURLKey(t *testing.T) { + for _, key := range parserOutboundURLKeys { + t.Run(key, func(t *testing.T) { + err := validateParserEngineOverrideURLs(map[string]string{ + key: "http://169.254.169.254/latest/meta-data", + }) + if err == nil { + t.Fatalf("expected %s to be rejected", key) + } + }) + } +} + +func TestValidateParserOverrideURLsIgnoresNonURLKeys(t *testing.T) { + if err := validateParserEngineOverrideURLs(map[string]string{ + "mineru_model_version": "pipeline", + }); err != nil { + t.Fatalf("unexpected error for non-URL override: %v", err) + } +} diff --git a/internal/application/service/vectorstore_healthcheck.go b/internal/application/service/vectorstore_healthcheck.go index 36eadee9..44f1caf9 100644 --- a/internal/application/service/vectorstore_healthcheck.go +++ b/internal/application/service/vectorstore_healthcheck.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "net" "net/http" "strings" "time" @@ -15,6 +14,7 @@ import ( "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/go-sql-driver/mysql" // MySQL driver for database/sql, used by Doris connection test _ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql "github.com/qdrant/go-client/qdrant" @@ -22,6 +22,7 @@ import ( "github.com/weaviate/weaviate-go-client/v5/weaviate" "github.com/weaviate/weaviate-go-client/v5/weaviate/auth" wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc" + "google.golang.org/grpc" ) const connectionTestTimeout = 10 * time.Second @@ -71,14 +72,14 @@ func testElasticsearchConnection(ctx context.Context, config types.ConnectionCon req.SetBasicAuth(config.Username, config.Password) } - client := &http.Client{ - Timeout: connectionTestTimeout, + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = connectionTestTimeout + client := secutils.NewSSRFSafeHTTPClient(clientCfg) + client.CheckRedirect = func(*http.Request, []*http.Request) error { // A health probe must not follow redirects: a malicious or compromised // endpoint could return a 302 to an internal address, defeating the // SSRF check that only validated the original (user-supplied) Addr. - CheckRedirect: func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse - }, + return http.ErrUseLastResponse } resp, err := client.Do(req) if err != nil { @@ -155,6 +156,9 @@ func testQdrantConnection(ctx context.Context, config types.ConnectionConfig) (s Port: port, APIKey: config.APIKey, UseTLS: config.UseTLS, + GrpcOptions: []grpc.DialOption{ + grpc.WithContextDialer(secutils.SSRFSafeGRPCDialer), + }, }) if err != nil { return "", errors.NewBadRequestError("failed to create qdrant client: invalid configuration") @@ -184,7 +188,7 @@ func testMilvusConnection(ctx context.Context, config types.ConnectionConfig) (s addr = "localhost:19530" } - conn, err := (&net.Dialer{}).DialContext(testCtx, "tcp", addr) + conn, err := secutils.SSRFSafeDialContext(testCtx, "tcp", addr) if err != nil { logger.Warnf(ctx, "Milvus connection test failed: %v", err) return "", errors.NewBadRequestError("failed to connect to milvus: connection refused or server unreachable") @@ -201,6 +205,9 @@ func testTencentVectorDBConnection(ctx context.Context, config types.ConnectionC client, err := tcvectordb.NewRpcClient(config.Addr, config.Username, config.APIKey, &tcvectordb.ClientOption{ ReadConsistency: tcvectordb.EventualConsistency, Timeout: connectionTestTimeout, + Transport: &secutils.SSRFValidatingRoundTripper{ + Base: secutils.NewSSRFSafeTransport(secutils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { logger.Warnf(ctx, "Tencent VectorDB connection test failed: %v", err) @@ -233,7 +240,8 @@ func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig) } weaviateCfg := weaviate.Config{ - Host: host, + Host: host, + ConnectionClient: secutils.NewSSRFSafeHTTPClient(secutils.DefaultSSRFSafeHTTPClientConfig()), GrpcConfig: &wgrpc.Config{ Host: grpcAddress, }, @@ -291,7 +299,8 @@ func testDorisConnection(ctx context.Context, config types.ConnectionConfig) (st cfg := mysql.NewConfig() cfg.User = config.Username cfg.Passwd = config.Password - cfg.Net = "tcp" + secutils.RegisterMySQLSSRFDialer() + cfg.Net = secutils.MySQLSSRFNetwork cfg.Addr = config.Addr cfg.DBName = database cfg.Timeout = 5 * time.Second diff --git a/internal/application/service/vectorstore_healthcheck_ssrf_test.go b/internal/application/service/vectorstore_healthcheck_ssrf_test.go new file mode 100644 index 00000000..2e23cf00 --- /dev/null +++ b/internal/application/service/vectorstore_healthcheck_ssrf_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestTestConnection_MilvusBlocksUnsafeAddrAtDialSink(t *testing.T) { + svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := svc.TestConnection(ctx, types.MilvusRetrieverEngineType, types.ConnectionConfig{ + Addr: "169.254.169.254:19530", + }) + if err == nil { + t.Fatal("expected milvus connectivity probe to reject unsafe address") + } + if !strings.Contains(err.Error(), "failed to connect to milvus") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestTestConnection_DorisBlocksUnsafeAddrAtDialSink(t *testing.T) { + svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := svc.TestConnection(ctx, types.DorisRetrieverEngineType, types.ConnectionConfig{ + Addr: "169.254.169.254:9030", + Database: "weknora", + Username: "root", + }) + if err == nil { + t.Fatal("expected doris connectivity probe to reject unsafe address") + } + if !strings.Contains(err.Error(), "failed to connect to doris") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/application/service/weknoracloud.go b/internal/application/service/weknoracloud.go index a6d6ce6e..0c838602 100644 --- a/internal/application/service/weknoracloud.go +++ b/internal/application/service/weknoracloud.go @@ -72,7 +72,9 @@ func (s *weKnoraCloudService) verifyCredentials(ctx context.Context, appID, appS logger.Infof(ctx, "credential verification request: method=GET url=%s app_id=%s request_id=%s ", healthURL, appID, requestID) - client := &http.Client{Timeout: 10 * time.Second} + clientCfg := utils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 10 * time.Second + client := utils.NewSSRFSafeHTTPClient(clientCfg) resp, err := client.Do(req) if err != nil { logger.Warnf(ctx, "credential verification HTTP failed: url=%s err=%v", healthURL, err) diff --git a/internal/container/engine_factory.go b/internal/container/engine_factory.go index 1567a8b7..c37f757d 100644 --- a/internal/container/engine_factory.go +++ b/internal/container/engine_factory.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "net" "strconv" "strings" "time" @@ -33,6 +34,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/tencent/vectordatabase-sdk-go/tcvectordb" ) @@ -58,6 +60,9 @@ func createEngineServiceFromStore( cfg *config.Config, auditSink openSearchRepo.AuditSink, ) (interfaces.RetrieveEngineService, error) { + if err := validateRuntimeVectorStoreAddresses(store); err != nil { + return nil, err + } switch store.EngineType { case types.PostgresRetrieverEngineType: return createPostgresEngine(store, db) @@ -82,6 +87,48 @@ func createEngineServiceFromStore( } } +// validateRuntimeVectorStoreAddresses is the final guard for persisted or +// imported vector-store rows. Create/test handlers validate the same fields at +// the input boundary, but runtime construction must not assume every stored row +// was written through those handlers. +func validateRuntimeVectorStoreAddresses(store types.VectorStore) error { + cc := store.ConnectionConfig + check := func(label, endpoint string) error { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return nil + } + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("%s failed SSRF validation: %w", label, err) + } + return nil + } + + switch store.EngineType { + case types.PostgresRetrieverEngineType, types.SQLiteRetrieverEngineType: + return nil + case types.ElasticsearchRetrieverEngineType, + types.OpenSearchRetrieverEngineType, + types.MilvusRetrieverEngineType, + types.TencentVectorDBRetrieverEngineType, + types.DorisRetrieverEngineType: + return check("vector store address", cc.Addr) + case types.QdrantRetrieverEngineType: + endpoint := cc.Host + if endpoint != "" && cc.Port != 0 { + endpoint = net.JoinHostPort(strings.Trim(cc.Host, "[]"), strconv.Itoa(cc.Port)) + } + return check("qdrant address", endpoint) + case types.WeaviateRetrieverEngineType: + if err := check("weaviate HTTP address", cc.Host); err != nil { + return err + } + return check("weaviate gRPC address", cc.GrpcAddress) + default: + return fmt.Errorf("vector store engine %q has no SSRF address policy", store.EngineType) + } +} + // createOpenSearchEngine builds an OpenSearch k-NN retrieve engine. Mirrors // createElasticsearchV8Engine but uses the driver's TLS-hardened client // constructor and injects the audit sink. NewRepository probes the cluster @@ -142,10 +189,16 @@ func isESv7(version string) bool { func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) { cc := store.ConnectionConfig + if err := utils.ValidateURLForSSRF(cc.Addr); err != nil { + return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err) + } client, err := elasticsearch.NewTypedClient(elasticsearch.Config{ Addresses: []string{cc.Addr}, Username: cc.Username, Password: cc.Password, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create elasticsearch v8 client: %w", err) @@ -156,10 +209,16 @@ func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (i func createElasticsearchV7Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) { cc := store.ConnectionConfig + if err := utils.ValidateURLForSSRF(cc.Addr); err != nil { + return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err) + } client, err := esv7.NewClient(esv7.Config{ Addresses: []string{cc.Addr}, Username: cc.Username, Password: cc.Password, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create elasticsearch v7 client: %w", err) @@ -176,10 +235,11 @@ func createQdrantEngine(store types.VectorStore) (interfaces.RetrieveEngineServi } client, err := qdrant.NewClient(&qdrant.Config{ - Host: cc.Host, - Port: port, - APIKey: cc.APIKey, - UseTLS: cc.UseTLS, + Host: cc.Host, + Port: port, + APIKey: cc.APIKey, + UseTLS: cc.UseTLS, + GrpcOptions: []grpc.DialOption{grpc.WithContextDialer(utils.SSRFSafeGRPCDialer)}, }) if err != nil { return nil, fmt.Errorf("create qdrant client: %w", err) @@ -205,8 +265,11 @@ func buildMilvusClientConfig(cc types.ConnectionConfig) milvusclient.ClientConfi } milvusCfg := milvusclient.ClientConfig{ - Address: addr, - DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)}, + Address: addr, + DialOptions: []grpc.DialOption{ + grpc.WithTimeout(5 * time.Second), + grpc.WithContextDialer(utils.SSRFSafeGRPCDialer), + }, } if cc.Username != "" { milvusCfg.Username = cc.Username @@ -236,7 +299,8 @@ func createWeaviateEngine(store types.VectorStore) (interfaces.RetrieveEngineSer } weaviateCfg := weaviate.Config{ - Host: host, + Host: host, + ConnectionClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()), GrpcConfig: &wgrpc.Config{ Host: grpcAddress, }, @@ -275,7 +339,8 @@ func createDorisEngine(store types.VectorStore) (interfaces.RetrieveEngineServic mc := mysql.NewConfig() mc.User = cc.Username mc.Passwd = cc.Password - mc.Net = "tcp" + utils.RegisterMySQLSSRFDialer() + mc.Net = utils.MySQLSSRFNetwork mc.Addr = cc.Addr mc.DBName = cc.Database mc.Params = map[string]string{"charset": "utf8mb4"} @@ -314,6 +379,9 @@ func createTencentVectorDBEngine(store types.VectorStore) (interfaces.RetrieveEn client, err := tcvectordb.NewRpcClient(cc.Addr, cc.Username, cc.APIKey, &tcvectordb.ClientOption{ ReadConsistency: tcvectordb.EventualConsistency, Timeout: 10 * time.Second, + Transport: &utils.SSRFValidatingRoundTripper{ + Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()), + }, }) if err != nil { return nil, fmt.Errorf("create tencent vectordb client: %w", err) diff --git a/internal/container/engine_factory_test.go b/internal/container/engine_factory_test.go index bd61c418..6114df4a 100644 --- a/internal/container/engine_factory_test.go +++ b/internal/container/engine_factory_test.go @@ -26,8 +26,8 @@ func TestBuildMilvusClientConfig_UsesDatabaseName(t *testing.T) { if cfg.DBName != "regdi_ram_haom1" { t.Fatalf("expected DBName to be propagated, got %q", cfg.DBName) } - if len(cfg.DialOptions) != 1 { - t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions)) + if len(cfg.DialOptions) != 2 { + t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions)) } } @@ -40,7 +40,7 @@ func TestBuildMilvusClientConfig_DefaultsAddressWhenMissing(t *testing.T) { if cfg.DBName != "" { t.Fatalf("expected empty DBName by default, got %q", cfg.DBName) } - if len(cfg.DialOptions) != 1 { - t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions)) + if len(cfg.DialOptions) != 2 { + t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions)) } } diff --git a/internal/container/ssrf_test.go b/internal/container/ssrf_test.go new file mode 100644 index 00000000..69173a50 --- /dev/null +++ b/internal/container/ssrf_test.go @@ -0,0 +1,43 @@ +package container + +import ( + "os" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + // Engine wiring tests intentionally use loopback httptest servers. + utils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost") + code := m.Run() + utils.SetSSRFWhitelistFromRaw("") + os.Exit(code) +} + +func TestValidateRuntimeVectorStoreAddressesRejectsUnsafeEndpoints(t *testing.T) { + tests := []types.VectorStore{ + {EngineType: types.QdrantRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "169.254.169.254", Port: 6334}}, + {EngineType: types.MilvusRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:19530"}}, + {EngineType: types.WeaviateRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "https://example.com", GrpcAddress: "169.254.169.254:50051"}}, + {EngineType: types.DorisRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:9030"}}, + {EngineType: types.TencentVectorDBRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:80"}}, + } + for _, store := range tests { + if err := validateRuntimeVectorStoreAddresses(store); err == nil { + t.Fatalf("expected %s unsafe address to be rejected", store.EngineType) + } + } +} + +func TestValidateRuntimeVectorStoreAddressesAllowsNonNetworkEngines(t *testing.T) { + for _, engineType := range []types.RetrieverEngineType{ + types.PostgresRetrieverEngineType, + types.SQLiteRetrieverEngineType, + } { + if err := validateRuntimeVectorStoreAddresses(types.VectorStore{EngineType: engineType}); err != nil { + t.Fatalf("unexpected %s validation error: %v", engineType, err) + } + } +} diff --git a/internal/handler/mcp_service.go b/internal/handler/mcp_service.go index 9d3255f4..55919eb2 100644 --- a/internal/handler/mcp_service.go +++ b/internal/handler/mcp_service.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/handler/dto" "github.com/Tencent/WeKnora/internal/logger" + mcpsecurity "github.com/Tencent/WeKnora/internal/mcp" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -74,6 +75,11 @@ func (h *MCPServiceHandler) CreateMCPService(c *gin.Context) { return } } + if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil { + logger.Warnf(ctx, "SSRF validation failed for MCP service configuration: %v", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } if err := h.mcpServiceService.CreateMCPService(ctx, &service); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_name": secutils.SanitizeForLog(service.Name)}) @@ -333,6 +339,11 @@ func (h *MCPServiceHandler) UpdateMCPService(c *gin.Context) { service.AdvancedConfig.RetryDelay = int(retryDelay) } } + if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil { + logger.Warnf(ctx, "SSRF validation failed for MCP service update: %v", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } if err := h.mcpServiceService.UpdateMCPService(ctx, &service); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)}) diff --git a/internal/handler/system.go b/internal/handler/system.go index c55e6cf4..69ff853d 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -28,8 +28,6 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "github.com/neo4j/neo4j-go-driver/v6/neo4j" ) @@ -863,59 +861,16 @@ func storageEndpointHost(endpoint string) string { return endpoint } -// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous -// address (cloud metadata, loopback, link-local). Unlike the stricter isSSRFSafeURL, -// this allows private IPs since MinIO is commonly deployed on internal networks. -// It also respects the SSRF_WHITELIST environment variable for whitelisted hosts. +// isBlockedStorageEndpoint keeps the legacy handler contract while delegating +// to the same fail-closed SSRF policy used by the storage clients themselves. +// Private storage endpoints must be explicitly whitelisted by an operator. func isBlockedStorageEndpoint(endpoint string) (bool, string) { - host := storageEndpointHost(endpoint) - if host == "" { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { return true, "无效的地址" } - - // Check SSRF whitelist first – whitelisted hosts bypass the block check. - if secutils.IsSSRFWhitelisted(host) { - return false, "" - } - - hostLower := strings.ToLower(host) - - blockedHosts := []string{ - "metadata.google.internal", - "metadata.tencentyun.com", - "metadata.aws.internal", - } - for _, bh := range blockedHosts { - if hostLower == bh { - return true, "该地址不允许访问" - } - } - - checkIP := func(ip net.IP) (bool, string) { - if ip.IsLoopback() { - return true, "不允许访问本地回环地址" - } - if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return true, "不允许访问链路本地地址" - } - if ip.IsUnspecified() { - return true, "无效的地址" - } - return false, "" - } - - if ip := net.ParseIP(host); ip != nil { - return checkIP(ip) - } - - ips, err := net.LookupIP(host) - if err != nil { - return false, "" - } - for _, ip := range ips { - if blocked, reason := checkIP(ip); blocked { - return blocked, reason - } + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return true, secutils.FormatSSRFError("存储 Endpoint", endpoint, err) } return false, "" } @@ -1029,15 +984,7 @@ func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *typ // If bucket does not exist, auto-create it if strings.Contains(errMsg, "does not exist") && cfg.BucketName != "" { logger.Info(ctx, "Storage check: bucket does not exist, attempting auto-creation", "bucket", cfg.BucketName) - minioClient, clientErr := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: cfg.UseSSL, - }) - if clientErr != nil { - c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to create MinIO client: %s", sanitizeStorageCheckError(clientErr))}}) - return - } - if mkErr := minioClient.MakeBucket(ctx, cfg.BucketName, minio.MakeBucketOptions{}); mkErr != nil { + if _, mkErr := file.NewMinioFileService(endpoint, accessKeyID, secretAccessKey, cfg.BucketName, cfg.UseSSL); mkErr != nil { logger.Error(ctx, "Storage check: failed to create bucket", "bucket", cfg.BucketName, "error", mkErr) c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to auto-create Bucket '%s': %s", cfg.BucketName, sanitizeStorageCheckError(mkErr))}}) return diff --git a/internal/handler/tenant.go b/internal/handler/tenant.go index 5f92a329..9bc99927 100644 --- a/internal/handler/tenant.go +++ b/internal/handler/tenant.go @@ -1737,5 +1737,15 @@ func validateParserEngineOutboundURLs(cfg *types.ParserEngineConfig) error { return fmt.Errorf("mineru_vlm_server_url failed SSRF validation: %v", err) } } + if odlURL := strings.TrimSpace(cfg.ODLHybridURL); odlURL != "" { + if err := secutils.ValidateURLForSSRF(odlURL); err != nil { + return fmt.Errorf("odl_hybrid_url failed SSRF validation: %v", err) + } + } + if endpoint := strings.TrimSpace(cfg.PaddleOCRVLEndpoint); endpoint != "" { + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("paddleocr_vl_endpoint failed SSRF validation: %v", err) + } + } return nil } diff --git a/internal/im/dingtalk/adapter.go b/internal/im/dingtalk/adapter.go index db574015..a39c45d0 100644 --- a/internal/im/dingtalk/adapter.go +++ b/internal/im/dingtalk/adapter.go @@ -26,7 +26,10 @@ import ( ) // httpClient is a shared HTTP client with a reasonable timeout for DingTalk API calls. -var httpClient = &http.Client{Timeout: 15 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 15 * time.Second, + MaxRedirects: 5, +}) // apiBaseURL is the DingTalk OpenAPI host. Overridable in tests. var apiBaseURL = "https://api.dingtalk.com" @@ -420,6 +423,9 @@ func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, r } func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, content string) error { + if err := secutils.ValidateURLForSSRF(webhookURL); err != nil { + return fmt.Errorf("dingtalk sessionWebhook rejected by SSRF policy: %w", err) + } body := map[string]interface{}{ "msgtype": "markdown", "markdown": map[string]string{ diff --git a/internal/im/dingtalk/download_test.go b/internal/im/dingtalk/download_test.go index a3a8f6d7..1eeb5dee 100644 --- a/internal/im/dingtalk/download_test.go +++ b/internal/im/dingtalk/download_test.go @@ -7,15 +7,24 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/Tencent/WeKnora/internal/im" ) +func useTestHTTPClient(t *testing.T) { + t.Helper() + original := httpClient + httpClient = &http.Client{Timeout: 5 * time.Second} + t.Cleanup(func() { httpClient = original }) +} + // TestDownloadFile_EndToEnd drives the full DownloadFile orchestration against a // fake DingTalk OpenAPI: access-token fetch → messageFiles/download (downloadCode // → temporary downloadUrl) → GET the temp URL for the bytes. This covers the HTTP // path that the pure-function unit tests deliberately skip (issue #1771). func TestDownloadFile_EndToEnd(t *testing.T) { + useTestHTTPClient(t) fileBytes := []byte("%PDF-1.7 fake product spec bytes") var downloadReq map[string]string @@ -89,6 +98,7 @@ func TestDownloadFile_EndToEnd(t *testing.T) { // TestDownloadFile_TempURLError verifies a non-200 from the temporary download // URL surfaces as an error rather than silently returning empty content. func TestDownloadFile_TempURLError(t *testing.T) { + useTestHTTPClient(t) var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -121,6 +131,7 @@ func TestDownloadFile_TempURLError(t *testing.T) { } func TestDownloadFile_SSRFRejected(t *testing.T) { + useTestHTTPClient(t) var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { diff --git a/internal/im/feishu/adapter.go b/internal/im/feishu/adapter.go index 0b744954..bc5902fb 100644 --- a/internal/im/feishu/adapter.go +++ b/internal/im/feishu/adapter.go @@ -39,7 +39,10 @@ import ( var _ im.StreamSender = (*Adapter)(nil) var _ im.FileDownloader = (*Adapter)(nil) -var httpClient = &http.Client{Timeout: 10 * time.Second} +var httpClient = utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ + Timeout: 10 * time.Second, + MaxRedirects: 5, +}) // Adapter implements im.Adapter for Feishu/Lark. type Adapter struct { diff --git a/internal/im/feishu/send_e2e_test.go b/internal/im/feishu/send_e2e_test.go index 4322519b..03aab714 100644 --- a/internal/im/feishu/send_e2e_test.go +++ b/internal/im/feishu/send_e2e_test.go @@ -8,10 +8,18 @@ import ( "strings" "sync" "testing" + "time" "github.com/Tencent/WeKnora/internal/im" ) +func useTestHTTPClient(t *testing.T) { + t.Helper() + original := httpClient + httpClient = &http.Client{Timeout: 5 * time.Second} + t.Cleanup(func() { httpClient = original }) +} + // testRegion returns a Lark-shaped region pointed at a fake Open Platform, so // the whole send path can be driven without touching the real cloud. Region // carries the host as a field precisely so this is possible. @@ -86,6 +94,7 @@ func (f *fakeOpenPlatform) sawPath(want string) bool { // SendReply must fetch a token from the region's cloud and reply under the // original message, carrying that token as a bearer credential. func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{} srv := httptest.NewServer(fake.handler()) defer srv.Close() @@ -130,6 +139,7 @@ func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) { // A group that rejects reply-in-thread (230071) must still receive the answer // via the plain send-message API, addressed to the chat. func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{replyCode: 230071} srv := httptest.NewServer(fake.handler()) defer srv.Close() @@ -160,6 +170,7 @@ func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) { // A non-fallback-eligible error must surface, not be silently swallowed. func TestSendReply_EndToEnd_HardErrorSurfaces(t *testing.T) { + useTestHTTPClient(t) fake := &fakeOpenPlatform{replyCode: 99991663} // app ticket invalid — not retryable srv := httptest.NewServer(fake.handler()) defer srv.Close() diff --git a/internal/im/qqbot/longconn.go b/internal/im/qqbot/longconn.go index ca79501e..7e468c3c 100644 --- a/internal/im/qqbot/longconn.go +++ b/internal/im/qqbot/longconn.go @@ -9,6 +9,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ws "github.com/gorilla/websocket" ) @@ -68,7 +69,9 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error { if err != nil { return err } - conn, _, err := ws.DefaultDialer.DialContext(ctx, gatewayURL, nil) + dialer := *ws.DefaultDialer + dialer.NetDialContext = secutils.SSRFSafeDialContext + conn, _, err := dialer.DialContext(ctx, gatewayURL, nil) if err != nil { return err } diff --git a/internal/im/telegram/adapter.go b/internal/im/telegram/adapter.go index cbb0d042..99859469 100644 --- a/internal/im/telegram/adapter.go +++ b/internal/im/telegram/adapter.go @@ -17,6 +17,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // Compile-time checks. @@ -72,8 +73,8 @@ func (a *Adapter) VerifyCallback(c *gin.Context) error { // telegramUpdate represents an incoming Telegram update (subset of fields). type telegramUpdate struct { - UpdateID int `json:"update_id"` - Message *telegramMsg `json:"message"` + UpdateID int `json:"update_id"` + Message *telegramMsg `json:"message"` } type telegramMsg struct { @@ -256,7 +257,10 @@ func (a *Adapter) editMessage(ctx context.Context, chatID, messageID, text, pars } // httpClient is a shared HTTP client with a reasonable timeout for Telegram API calls. -var httpClient = &http.Client{Timeout: 15 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 15 * time.Second, + MaxRedirects: 5, +}) // callAPI calls the Telegram Bot API, discarding the result. func (a *Adapter) callAPI(ctx context.Context, method string, body interface{}) error { @@ -311,7 +315,7 @@ const minEditInterval = 500 * time.Millisecond type streamState struct { mu sync.Mutex content strings.Builder - msgID string // Telegram message ID of the "thinking" message + msgID string // Telegram message ID of the "thinking" message chatID string lastEdit time.Time // last successful editMessageText timestamp createdAt time.Time // for orphan stream detection diff --git a/internal/im/telegram/longconn.go b/internal/im/telegram/longconn.go index 36373677..7d85d5e5 100644 --- a/internal/im/telegram/longconn.go +++ b/internal/im/telegram/longconn.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // MessageHandler is called when an IM message is received via long polling. @@ -26,9 +27,12 @@ type LongConnClient struct { // NewLongConnClient creates a Telegram long-polling client. func NewLongConnClient(botToken string, handler MessageHandler) *LongConnClient { return &LongConnClient{ - botToken: botToken, - handler: handler, - httpClient: &http.Client{Timeout: 35 * time.Second}, + botToken: botToken, + handler: handler, + httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 35 * time.Second, + MaxRedirects: 5, + }), } } diff --git a/internal/im/wechat/adapter.go b/internal/im/wechat/adapter.go index 4a3c3fdb..52411069 100644 --- a/internal/im/wechat/adapter.go +++ b/internal/im/wechat/adapter.go @@ -24,6 +24,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" ) @@ -37,7 +38,10 @@ const ( channelVersion = "weknora-1.0.0" ) -var ilinkHTTPClient = &http.Client{Timeout: 30 * time.Second} +var ilinkHTTPClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 30 * time.Second, + MaxRedirects: 5, +}) // BuildCDNDownloadURL constructs a CDN download URL from an encrypt_query_param. func BuildCDNDownloadURL(encryptQueryParam string) string { @@ -145,6 +149,9 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io if msg.FileKey == "" { return nil, "", fmt.Errorf("no file URL in message") } + if err := secutils.ValidateURLForSSRF(msg.FileKey); err != nil { + return nil, "", fmt.Errorf("file URL rejected by SSRF policy: %w", err) + } fileName := msg.FileName if fileName == "" { diff --git a/internal/im/wechat/longpoll.go b/internal/im/wechat/longpoll.go index b7358c3b..e892279e 100644 --- a/internal/im/wechat/longpoll.go +++ b/internal/im/wechat/longpoll.go @@ -22,13 +22,14 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) const ( - longPollTimeout = 35 * time.Second - longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout - reconnectBaseDelay = 1 * time.Second - reconnectMaxDelay = 30 * time.Second + longPollTimeout = 35 * time.Second + longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout + reconnectBaseDelay = 1 * time.Second + reconnectMaxDelay = 30 * time.Second maxReconnectAttempts = -1 // infinite ) @@ -50,7 +51,10 @@ func NewLongPollClient(botToken, ilinkBotID string, handler func(ctx context.Con botToken: botToken, ilinkBotID: ilinkBotID, handler: handler, - httpClient: &http.Client{Timeout: longPollHTTPTimeout}, + httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: longPollHTTPTimeout, + MaxRedirects: 5, + }), } } @@ -316,25 +320,25 @@ func pollReconnectDelay(attempt int) time.Duration { // ── iLink API response types (matches proto: GetUpdatesResp, WeixinMessage) ── type getUpdatesResponse struct { - Ret int `json:"ret"` - ErrCode int `json:"errcode"` - ErrMsg string `json:"errmsg"` - Msgs []weixinMessage `json:"msgs"` - GetUpdatesBuf string `json:"get_updates_buf"` + Ret int `json:"ret"` + ErrCode int `json:"errcode"` + ErrMsg string `json:"errmsg"` + Msgs []weixinMessage `json:"msgs"` + GetUpdatesBuf string `json:"get_updates_buf"` } type weixinMessage struct { - Seq int `json:"seq"` - MessageID int64 `json:"message_id"` - FromUserID string `json:"from_user_id"` - ToUserID string `json:"to_user_id"` - ClientID string `json:"client_id"` - CreateTimeMs int64 `json:"create_time_ms"` - SessionID string `json:"session_id"` - MessageType int `json:"message_type"` // 1=USER, 2=BOT - MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH - ItemList []messageItem `json:"item_list"` - ContextToken string `json:"context_token"` + Seq int `json:"seq"` + MessageID int64 `json:"message_id"` + FromUserID string `json:"from_user_id"` + ToUserID string `json:"to_user_id"` + ClientID string `json:"client_id"` + CreateTimeMs int64 `json:"create_time_ms"` + SessionID string `json:"session_id"` + MessageType int `json:"message_type"` // 1=USER, 2=BOT + MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH + ItemList []messageItem `json:"item_list"` + ContextToken string `json:"context_token"` } type messageItem struct { diff --git a/internal/im/wechat/qrcode.go b/internal/im/wechat/qrcode.go index 176de300..3bed7017 100644 --- a/internal/im/wechat/qrcode.go +++ b/internal/im/wechat/qrcode.go @@ -1,8 +1,9 @@ // QR code login flow for WeChat iLink Bot API. // // API endpoints (all relative to the iLink base URL): -// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content} -// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...} +// +// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content} +// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...} // // Flow: // 1. Call GetLoginQRCode to obtain a QR code URL and opaque qrcode token @@ -19,6 +20,8 @@ import ( "net/http" "net/url" "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" ) // QRCodeResult holds the result of requesting a login QR code. @@ -46,7 +49,10 @@ type QRCodeService struct { // NewQRCodeService creates a new QR code service. func NewQRCodeService() *QRCodeService { return &QRCodeService{ - client: &http.Client{Timeout: pollTimeout}, + client: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: pollTimeout, + MaxRedirects: 5, + }), } } @@ -76,7 +82,7 @@ func (s *QRCodeService) GetLoginQRCode(ctx context.Context) (*QRCodeResult, erro } var result struct { - QRCode string `json:"qrcode"` + QRCode string `json:"qrcode"` QRCodeImgContent string `json:"qrcode_img_content"` } if err := json.Unmarshal(body, &result); err != nil { diff --git a/internal/im/wecom/longconn.go b/internal/im/wecom/longconn.go index b545c90d..3ecd6cf4 100644 --- a/internal/im/wecom/longconn.go +++ b/internal/im/wecom/longconn.go @@ -22,6 +22,7 @@ import ( "github.com/Tencent/WeKnora/internal/im" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ws "github.com/gorilla/websocket" ) @@ -365,7 +366,9 @@ func (c *LongConnClient) sendStreamFrame(incoming *im.IncomingMessage, streamID, } func (c *LongConnClient) connectAndRun(ctx context.Context) error { - conn, _, err := ws.DefaultDialer.DialContext(ctx, c.endpoint, nil) + dialer := *ws.DefaultDialer + dialer.NetDialContext = secutils.SSRFSafeDialContext + conn, _, err := dialer.DialContext(ctx, c.endpoint, nil) if err != nil { return fmt.Errorf("dial: %w", err) } diff --git a/internal/im/wecom/webhook_adapter.go b/internal/im/wecom/webhook_adapter.go index 1166eb85..fe37cdb4 100644 --- a/internal/im/wecom/webhook_adapter.go +++ b/internal/im/wecom/webhook_adapter.go @@ -37,7 +37,10 @@ import ( "github.com/gin-gonic/gin" ) -var httpClient = &http.Client{Timeout: 30 * time.Second} +var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 30 * time.Second, + MaxRedirects: 5, +}) const defaultAPIBaseURL = "https://qyapi.weixin.qq.com" diff --git a/internal/infrastructure/docparser/http_parser.go b/internal/infrastructure/docparser/http_parser.go index 7d0267d8..aeca68e6 100644 --- a/internal/infrastructure/docparser/http_parser.go +++ b/internal/infrastructure/docparser/http_parser.go @@ -14,6 +14,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" ) const ( @@ -62,16 +63,17 @@ type HTTPDocumentReader struct { } func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) { + baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/") + if baseURL != "" { + if err := secutils.ValidateURLForSSRF(baseURL); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } + } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 5 * time.Minute p := &HTTPDocumentReader{ - baseURL: strings.TrimSuffix(baseURL, "/"), - client: &http.Client{ - Timeout: 5 * time.Minute, - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, - }, + baseURL: baseURL, + client: secutils.NewSSRFSafeHTTPClient(clientCfg), } if p.baseURL != "" { logger.Infof(context.Background(), "INFO: HTTP docreader base URL: %s", p.baseURL) @@ -86,9 +88,15 @@ func (p *HTTPDocumentReader) base() string { } func (p *HTTPDocumentReader) Reconnect(addr string) error { + addr = strings.TrimSuffix(strings.TrimSpace(addr), "/") + if addr != "" { + if err := secutils.ValidateURLForSSRF(addr); err != nil { + return fmt.Errorf("docreader address failed SSRF validation: %w", err) + } + } p.mu.Lock() defer p.mu.Unlock() - p.baseURL = strings.TrimSuffix(addr, "/") + p.baseURL = addr logger.Infof(context.Background(), "INFO: HTTP docreader base URL set to %s", p.baseURL) return nil } @@ -122,6 +130,9 @@ func (p *HTTPDocumentReader) ListEngines(ctx context.Context, overrides map[stri if base == "" { return nil, errNotConnected } + if err := secutils.ValidateURLForSSRF(base); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } body := httpListEnginesRequest{ConfigOverrides: overrides} jsonBody, err := json.Marshal(body) @@ -187,6 +198,9 @@ func (p *HTTPDocumentReader) Read(ctx context.Context, req *types.ReadRequest) ( if base == "" { return nil, errNotConnected } + if err := secutils.ValidateURLForSSRF(base); err != nil { + return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err) + } body := httpReadRequest{ FileName: req.FileName, diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go index 9beba038..98cfc013 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go @@ -51,6 +51,9 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques if c.token == "" { return &types.ReadResult{Error: "PaddleOCR-VL Cloud token is not configured"}, nil } + if err := utils.ValidateURLForSSRF(c.baseURL); err != nil { + return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL Cloud base URL blocked by SSRF policy: %v", err)}, nil + } content := req.FileContent if len(content) == 0 { diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go index 5ab7fdcb..313d3b74 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go @@ -44,6 +44,9 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (* if c.endpoint == "" { return &types.ReadResult{Error: "PaddleOCR-VL endpoint is not configured"}, nil } + if err := utils.ValidateURLForSSRF(c.endpoint); err != nil { + return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL endpoint blocked by SSRF policy: %v", err)}, nil + } content := req.FileContent if len(content) == 0 { @@ -157,7 +160,10 @@ func (c *PaddleOCRVLReader) callLayoutParsing( } httpReq.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: paddleOCRVLTimeout} + client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ + Timeout: paddleOCRVLTimeout, + MaxRedirects: 5, + }) resp, err := client.Do(httpReq) if err != nil { return "", nil, fmt.Errorf("HTTP request: %w", err) @@ -265,6 +271,9 @@ func PingPaddleOCRVL(endpoint string) (bool, string) { if endpoint == "" { return false, "未配置 PaddleOCR-VL 端点" } + if err := utils.ValidateURLForSSRF(endpoint); err != nil { + return false, fmt.Sprintf("PaddleOCR-VL 端点未通过 SSRF 校验: %v", err) + } client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ Timeout: 5 * time.Second, MaxRedirects: 5, diff --git a/internal/infrastructure/docparser/weknoracloud_http_reader.go b/internal/infrastructure/docparser/weknoracloud_http_reader.go index 866085f7..f19ef710 100644 --- a/internal/infrastructure/docparser/weknoracloud_http_reader.go +++ b/internal/infrastructure/docparser/weknoracloud_http_reader.go @@ -15,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/utils" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" ) @@ -39,20 +40,15 @@ func NewWeKnoraCloudSignedDocumentReader(appID, apiKey string) (*WeKnoraCloudSig if apiKey == "" { return nil, fmt.Errorf("WeKnoraCloud apiKey is required") } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = 500 * time.Minute return &WeKnoraCloudSignedDocumentReader{ appID: appID, apiKey: apiKey, initialPollInterval: 500 * time.Millisecond, maxPollInterval: 10 * time.Second, pollTimeout: 20 * time.Minute, - client: &http.Client{ - Timeout: 500 * time.Minute, - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, - }, + client: secutils.NewSSRFSafeHTTPClient(clientCfg), }, nil } diff --git a/internal/infrastructure/web_search/baidu.go b/internal/infrastructure/web_search/baidu.go index 3b2ce698..3a8490e1 100644 --- a/internal/infrastructure/web_search/baidu.go +++ b/internal/infrastructure/web_search/baidu.go @@ -39,8 +39,9 @@ func NewBaiduProvider(params types.WebSearchProviderParameters) (interfaces.WebS if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Baidu provider") } - client := &http.Client{ - Timeout: defaultBaiduTimeout, + client, err := NewSearchHTTPClient(defaultBaiduTimeout, params.ProxyURL) + if err != nil { + return nil, fmt.Errorf("create Baidu HTTP client: %w", err) } return &BaiduProvider{ client: client, diff --git a/internal/infrastructure/web_search/ollama.go b/internal/infrastructure/web_search/ollama.go index c555597a..b4e3fa8f 100644 --- a/internal/infrastructure/web_search/ollama.go +++ b/internal/infrastructure/web_search/ollama.go @@ -35,8 +35,9 @@ func NewOllamaProvider(params types.WebSearchProviderParameters) (interfaces.Web if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Ollama provider") } - client := &http.Client{ - Timeout: defaultOllamaTimeout, + client, err := NewSearchHTTPClient(defaultOllamaTimeout, params.ProxyURL) + if err != nil { + return nil, fmt.Errorf("create Ollama HTTP client: %w", err) } return &OllamaProvider{ client: client, diff --git a/internal/infrastructure/web_search/proxy.go b/internal/infrastructure/web_search/proxy.go index d7bc0689..3dc38816 100644 --- a/internal/infrastructure/web_search/proxy.go +++ b/internal/infrastructure/web_search/proxy.go @@ -19,18 +19,6 @@ func ValidateProxyURL(proxyURL string) error { return utils.ValidateURLForSSRF(proxyURL) } -func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error { - return func(req *http.Request, via []*http.Request) error { - if len(via) >= maxRedirects { - return fmt.Errorf("stopped after %d redirects", maxRedirects) - } - if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil { - return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err) - } - return nil - } -} - // NewSearchHTTPClient builds an http.Client for outbound web search requests. // It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and // redirect validation consistent with utils.NewSSRFSafeHTTPClient. @@ -60,9 +48,6 @@ func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client, } cfg := utils.DefaultSSRFSafeHTTPClientConfig() - return &http.Client{ - Timeout: timeout, - Transport: t, - CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects), - }, nil + cfg.Timeout = timeout + return utils.NewSSRFSafeHTTPClientWithTransport(cfg, t), nil } diff --git a/internal/infrastructure/web_search/proxy_test.go b/internal/infrastructure/web_search/proxy_test.go index 46f318d1..9a2fc121 100644 --- a/internal/infrastructure/web_search/proxy_test.go +++ b/internal/infrastructure/web_search/proxy_test.go @@ -59,10 +59,13 @@ func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) { } func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) { - h := ssrfSafeRedirect(10) + client, err := NewSearchHTTPClient(0, "") + if err != nil { + t.Fatal(err) + } req := &http.Request{URL: mustParse(t, "ftp://evil.com/")} // len(via)==0 so first redirect - err := h(req, nil) + err = client.CheckRedirect(req, nil) if err == nil { t.Fatal("expected error for ftp redirect") } diff --git a/internal/mcp/client.go b/internal/mcp/client.go index 63b2bb94..c49962f9 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -12,6 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/mark3labs/mcp-go/client" "github.com/mark3labs/mcp-go/client/transport" "github.com/mark3labs/mcp-go/mcp" @@ -147,15 +148,22 @@ func asOAuthRequired(err error) *OAuthRequiredError { // NewMCPClient creates a new MCP client based on the transport type func NewMCPClient(config *ClientConfig) (MCPClient, error) { + if config == nil || config.Service == nil { + return nil, fmt.Errorf("MCP client config and service are required") + } + if err := ValidateServiceOutboundURLs(config.Service); err != nil { + return nil, err + } + // Create HTTP client with timeout timeout := 30 * time.Second if config.Service.AdvancedConfig != nil && config.Service.AdvancedConfig.Timeout > 0 { timeout = time.Duration(config.Service.AdvancedConfig.Timeout) * time.Second } - httpClient := &http.Client{ - Timeout: timeout, - } + clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + clientCfg.Timeout = timeout + httpClient := secutils.NewSSRFSafeHTTPClient(clientCfg) // Build headers headers := make(map[string]string) diff --git a/internal/mcp/oauth_manager.go b/internal/mcp/oauth_manager.go index 761e5fd1..0791e4aa 100644 --- a/internal/mcp/oauth_manager.go +++ b/internal/mcp/oauth_manager.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/mark3labs/mcp-go/client/transport" "github.com/redis/go-redis/v9" ) @@ -52,12 +53,18 @@ func (m *OAuthManager) newHandler( if service.URL == nil || *service.URL == "" { return nil, fmt.Errorf("MCP service URL is required for OAuth") } + if err := ValidateServiceOutboundURLs(service); err != nil { + return nil, err + } + httpCfg := secutils.DefaultSSRFSafeHTTPClientConfig() + httpCfg.Timeout = 30 * time.Second cfg := transport.OAuthConfig{ RedirectURI: redirectURI, Scopes: service.AuthConfig.Scopes, TokenStore: newDBTokenStore(m.repo, tenantID, principal, service.ID), PKCEEnabled: true, AuthServerMetadataURL: service.AuthConfig.AuthServerMetadataURL, + HTTPClient: secutils.NewSSRFSafeHTTPClient(httpCfg), } if existing, err := m.repo.GetClient(ctx, tenantID, service.ID); err == nil && existing != nil { cfg.ClientID = existing.ClientID diff --git a/internal/mcp/security.go b/internal/mcp/security.go new file mode 100644 index 00000000..d614e3ff --- /dev/null +++ b/internal/mcp/security.go @@ -0,0 +1,36 @@ +package mcp + +import ( + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// ValidateServiceOutboundURLs validates every URL that the MCP transport or +// OAuth discovery flow may contact. It is intentionally called both at +// persistence boundaries and immediately before client construction so stale +// or imported rows cannot bypass the current SSRF policy. +func ValidateServiceOutboundURLs(service *types.MCPService) error { + if service == nil { + return fmt.Errorf("MCP service is required") + } + if service.URL != nil { + serviceURL := strings.TrimSpace(*service.URL) + if serviceURL != "" { + if err := secutils.ValidateURLForSSRF(serviceURL); err != nil { + return fmt.Errorf("MCP service URL failed SSRF validation: %w", err) + } + } + } + if service.AuthConfig != nil { + metadataURL := strings.TrimSpace(service.AuthConfig.AuthServerMetadataURL) + if metadataURL != "" { + if err := secutils.ValidateURLForSSRF(metadataURL); err != nil { + return fmt.Errorf("MCP OAuth metadata URL failed SSRF validation: %w", err) + } + } + } + return nil +} diff --git a/internal/mcp/security_test.go b/internal/mcp/security_test.go new file mode 100644 index 00000000..b79ecc8c --- /dev/null +++ b/internal/mcp/security_test.go @@ -0,0 +1,31 @@ +package mcp + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestValidateServiceOutboundURLsRejectsOAuthMetadataSSRF(t *testing.T) { + serviceURL := "https://example.com/mcp" + err := ValidateServiceOutboundURLs(&types.MCPService{ + URL: &serviceURL, + AuthConfig: &types.MCPAuthConfig{ + AuthServerMetadataURL: "http://169.254.169.254/latest/meta-data", + }, + }) + if err == nil { + t.Fatal("expected OAuth metadata URL to be rejected") + } +} + +func TestNewMCPClientRejectsStoredUnsafeURL(t *testing.T) { + serviceURL := "http://127.0.0.1:8080/mcp" + _, err := NewMCPClient(&ClientConfig{Service: &types.MCPService{ + URL: &serviceURL, + TransportType: types.MCPTransportHTTPStreamable, + }}) + if err == nil { + t.Fatal("expected stored unsafe MCP URL to be rejected at client construction") + } +} diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 9364cb71..02b93d0f 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -72,16 +72,16 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { } } + // The SDK must use the same SSRF-safe transport as the raw HTTP paths. + // Constructor-time URL validation alone cannot prevent DNS rebinding or a + // later redirect to an internal address. + sdkHTTPClient := rawHTTPClient // 如果指定了 CustomHeaders,则给 SDK 使用的 HTTPClient 挂一层 RoundTripper, // 在每个请求上自动注入这些 header(raw HTTP 路径会在发送前单独处理)。 if len(chatConfig.CustomHeaders) > 0 { - if httpClient, ok := config.HTTPClient.(*http.Client); ok { - config.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, chatConfig.CustomHeaders) - } else { - // SDK 默认未显式设置时 HTTPClient 为 nil,此时构造一个新的注入了 header 的 client。 - config.HTTPClient = secutils.WrapHTTPClientWithHeaders(nil, chatConfig.CustomHeaders) - } + sdkHTTPClient = secutils.WrapHTTPClientWithHeaders(sdkHTTPClient, chatConfig.CustomHeaders) } + config.HTTPClient = sdkHTTPClient modelName := chatConfig.ModelName if chatConfig.ExtraConfig != nil { diff --git a/internal/models/chat/transport.go b/internal/models/chat/transport.go index 8496bd1f..91b3dc26 100644 --- a/internal/models/chat/transport.go +++ b/internal/models/chat/transport.go @@ -48,12 +48,15 @@ func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, cont // Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout) // rather than http.Client.Timeout, so streaming calls are not prematurely terminated. // Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer. -var rawHTTPClient = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: secutils.SSRFSafeDialContext, - TLSHandshakeTimeout: 10 * time.Second, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, +var rawHTTPTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: secutils.SSRFSafeDialContext, + TLSHandshakeTimeout: 10 * time.Second, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 5, } + +var rawHTTPClient = secutils.NewSSRFSafeHTTPClientWithTransport( + secutils.SSRFSafeHTTPClientConfig{Timeout: 0, MaxRedirects: 10}, + rawHTTPTransport, +) diff --git a/internal/models/embedding/transport_test.go b/internal/models/embedding/transport_test.go index 551406a6..5ea3358a 100644 --- a/internal/models/embedding/transport_test.go +++ b/internal/models/embedding/transport_test.go @@ -1,9 +1,12 @@ package embedding import ( + "net/http" "strings" "testing" "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" ) func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) { @@ -15,10 +18,18 @@ func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) { if first == second { t.Fatal("expected distinct HTTP clients") } - if first.Transport != second.Transport { - t.Fatal("expected embedding HTTP clients to share a transport") + firstGuard, ok := first.Transport.(*secutils.SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating transport, got %T", first.Transport) } - if first.Transport != sharedEmbeddingHTTPTransport { + secondGuard, ok := second.Transport.(*secutils.SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating transport, got %T", second.Transport) + } + if firstGuard.Base != secondGuard.Base { + t.Fatal("expected embedding HTTP clients to share a base transport") + } + if firstGuard.Base != http.RoundTripper(sharedEmbeddingHTTPTransport) { t.Fatal("expected embedding HTTP client to use the shared transport") } if first.Timeout != firstTimeout { diff --git a/internal/utils/extraheaders.go b/internal/utils/extraheaders.go index fd504466..b32d54f8 100644 --- a/internal/utils/extraheaders.go +++ b/internal/utils/extraheaders.go @@ -75,7 +75,7 @@ func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) * return client } if client == nil { - client = &http.Client{} + client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig()) } base := client.Transport if base == nil { diff --git a/internal/utils/extraheaders_test.go b/internal/utils/extraheaders_test.go index e24f972a..5f4ecf33 100644 --- a/internal/utils/extraheaders_test.go +++ b/internal/utils/extraheaders_test.go @@ -50,6 +50,9 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) { } func TestWrapHTTPClientWithHeaders(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost") + ResetSSRFWhitelistForTest() + t.Cleanup(ResetSSRFWhitelistForTest) gotTrace := "" gotAuth := "" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/utils/httputil.go b/internal/utils/httputil.go index b1d3ed0a..9fd05cca 100644 --- a/internal/utils/httputil.go +++ b/internal/utils/httputil.go @@ -8,7 +8,10 @@ import ( "time" ) -var defaultHTTPClient = &http.Client{Timeout: 60 * time.Second} +var defaultHTTPClient = NewSSRFSafeHTTPClient(SSRFSafeHTTPClientConfig{ + Timeout: 60 * time.Second, + MaxRedirects: 10, +}) // DownloadBytes fetches the content at the given HTTP(S) URL and returns the // raw bytes. It reuses a package-level http.Client with a 60-second timeout. @@ -16,6 +19,9 @@ func DownloadBytes(url string) ([]byte, error) { if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { return nil, fmt.Errorf("unsupported URL scheme: %s", url) } + if err := ValidateURLForSSRF(url); err != nil { + return nil, fmt.Errorf("URL rejected by SSRF policy: %w", err) + } resp, err := defaultHTTPClient.Get(url) if err != nil { return nil, fmt.Errorf("HTTP GET: %w", err) diff --git a/internal/utils/mysql_ssrf.go b/internal/utils/mysql_ssrf.go new file mode 100644 index 00000000..e45dfc90 --- /dev/null +++ b/internal/utils/mysql_ssrf.go @@ -0,0 +1,27 @@ +package utils + +import ( + "context" + "net" + "sync" + + "github.com/go-sql-driver/mysql" +) + +// MySQLSSRFNetwork is the mysql driver network name registered by +// RegisterMySQLSSRFDialer. DSNs must set cfg.Net to this value so every +// database/sql open uses SSRFSafeDialContext at the final TCP sink. +const MySQLSSRFNetwork = "tcp-weknora-ssrf" + +var mysqlSSRFDialerOnce sync.Once + +// RegisterMySQLSSRFDialer installs a go-sql-driver/mysql dialer that routes +// connections through SSRFSafeDialContext. Safe to call from multiple packages; +// registration happens at most once per process. +func RegisterMySQLSSRFDialer() { + mysqlSSRFDialerOnce.Do(func() { + mysql.RegisterDialContext(MySQLSSRFNetwork, func(ctx context.Context, addr string) (net.Conn, error) { + return SSRFSafeDialContext(ctx, "tcp", addr) + }) + }) +} diff --git a/internal/utils/security.go b/internal/utils/security.go index 5b8b4157..df642778 100644 --- a/internal/utils/security.go +++ b/internal/utils/security.go @@ -254,6 +254,26 @@ var restrictedIPv4Ranges = []*net.IPNet{ mustParseCIDR("172.20.0.0/16"), } +// restrictedPorts contains non-HTTP service ports that user-controlled URLs +// must not reach. It is checked both during URL validation and again at dial +// time so dynamically discovered URLs cannot bypass the input boundary. +var restrictedPorts = map[string]bool{ + "22": true, // SSH + "23": true, // Telnet + "25": true, // SMTP + "445": true, // SMB + "3389": true, // RDP + "5432": true, // PostgreSQL + "3306": true, // MySQL + "6379": true, // Redis + "27017": true, // MongoDB + "9200": true, // Elasticsearch + "2379": true, // etcd + "2380": true, // etcd + "8500": true, // Consul + "4001": true, // etcd (old) +} + // mustParseCIDR parses a CIDR string and panics on error func mustParseCIDR(s string) *net.IPNet { _, ipNet, err := net.ParseCIDR(s) @@ -459,27 +479,8 @@ func isSSRFSafeURL(rawURL string) (bool, string) { // Check for suspicious port numbers port := parsed.Port() - if port != "" { - // Block common internal service ports - blockedPorts := map[string]bool{ - "22": true, // SSH - "23": true, // Telnet - "25": true, // SMTP - "445": true, // SMB - "3389": true, // RDP - "5432": true, // PostgreSQL - "3306": true, // MySQL - "6379": true, // Redis - "27017": true, // MongoDB - "9200": true, // Elasticsearch - "2379": true, // etcd - "2380": true, // etcd - "8500": true, // Consul - "4001": true, // etcd (old) - } - if blockedPorts[port] { - return false, fmt.Sprintf("port %s is blocked for security reasons", port) - } + if restrictedPorts[port] { + return false, fmt.Sprintf("port %s is blocked for security reasons", port) } return true, "" @@ -818,15 +819,36 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request) if redirectHost != "" && IsSSRFWhitelisted(redirectHost) { return nil } - redirectURL := req.URL.String() - if safe, reason := isSSRFSafeURL(redirectURL); !safe { - return fmt.Errorf("%w: %s", ErrSSRFRedirectBlocked, reason) + if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil { + return fmt.Errorf("%w: %w", ErrSSRFRedirectBlocked, err) } return nil } } +// SSRFValidatingRoundTripper enforces the URL policy for every outbound +// request, including URLs discovered at runtime by SDKs (for example OAuth +// metadata) that never passed through an application handler. Dial-time checks +// remain necessary to pin DNS answers and cover transports that cannot accept +// this wrapper directly. +type SSRFValidatingRoundTripper struct { + Base http.RoundTripper +} + +func (t *SSRFValidatingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req == nil || req.URL == nil { + return nil, fmt.Errorf("outbound request blocked: request URL is required") + } + if t == nil || t.Base == nil { + return nil, fmt.Errorf("outbound request blocked: base transport is required") + } + if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil { + return nil, fmt.Errorf("outbound request blocked by SSRF policy: %w", err) + } + return t.Base.RoundTrip(req) +} + // NewSSRFSafeHTTPClientWithTransport wraps a caller-supplied transport in an // *http.Client carrying the given timeout and the SSRF-aware redirect policy. // Pass a transport from NewSSRFSafeTransport (optionally shared across clients) @@ -834,9 +856,12 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request) func NewSSRFSafeHTTPClientWithTransport( config SSRFSafeHTTPClientConfig, transport http.RoundTripper, ) *http.Client { + if transport == nil { + transport = NewSSRFSafeTransport(config) + } return &http.Client{ Timeout: config.Timeout, - Transport: transport, + Transport: &SSRFValidatingRoundTripper{Base: transport}, CheckRedirect: newSSRFCheckRedirect(config.MaxRedirects), } } @@ -849,12 +874,18 @@ func NewSSRFSafeHTTPClient(config SSRFSafeHTTPClientConfig) *http.Client { return NewSSRFSafeHTTPClientWithTransport(config, NewSSRFSafeTransport(config)) } +// SSRFSafeGRPCDialer is compatible with grpc.WithContextDialer and pins DNS +// answers the same way as SSRFSafeDialContext. +func SSRFSafeGRPCDialer(ctx context.Context, addr string) (net.Conn, error) { + return SSRFSafeDialContext(ctx, "tcp", addr) +} + // SSRFSafeDialContext is a custom dial function that validates the resolved IP addresses // before establishing a connection. This provides an additional layer of SSRF protection // against DNS rebinding attacks during the connection phase. func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, error) { // Parse host and port - host, _, err := net.SplitHostPort(addr) + host, port, err := net.SplitHostPort(addr) if err != nil { return nil, fmt.Errorf("invalid address %s: %w", addr, err) } @@ -870,6 +901,9 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } return dialer.DialContext(ctx, network, addr) } + if restrictedPorts[port] { + return nil, fmt.Errorf("connection blocked: port %s is restricted", port) + } // Check if the host is a restricted hostname hostLower := strings.ToLower(host) @@ -884,11 +918,17 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } } - // Resolve the hostname to IP addresses + // Resolve the hostname once, validate every answer, and then dial one of + // those exact IPs. Dialing the original hostname here would make the + // standard dialer resolve it a second time, leaving a DNS-rebinding window + // between validation and connection establishment. ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, fmt.Errorf("DNS resolution failed for %s: %w", host, err) } + if len(ips) == 0 { + return nil, fmt.Errorf("DNS resolution returned no addresses for %s", host) + } // Validate all resolved IPs for _, ipAddr := range ips { @@ -897,13 +937,22 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e } } - // If we get here, all IPs are safe. Connect using the standard dialer. - // We dial the original address so that proper connection routing happens. + // If we get here, all IPs are safe. Pin the connection to the validated DNS + // answers; TLS still uses the request hostname for SNI/certificate checks. dialer := &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, } - return dialer.DialContext(ctx, network, addr) + var lastErr error + for _, ipAddr := range ips { + pinnedAddr := net.JoinHostPort(ipAddr.IP.String(), port) + conn, dialErr := dialer.DialContext(ctx, network, pinnedAddr) + if dialErr == nil { + return conn, nil + } + lastErr = dialErr + } + return nil, fmt.Errorf("failed to connect to validated addresses for %s: %w", host, lastErr) } // --------------------------------------------------------------------------- @@ -987,6 +1036,7 @@ func loadSSRFWhitelist() *ssrfWhitelistConfig { // applySSRFWhitelist for the canonical merge logic. func SetSSRFWhitelistFromRaw(raw string) { ssrfWhitelistAtomic.Store(parseSSRFWhitelistRaw(raw)) + invalidateSSRFOutboundValidationCache() } // parseSSRFWhitelistRaw parses a comma-separated whitelist string into @@ -1165,6 +1215,7 @@ func ResetSSRFWhitelistForTest() { ssrfWhitelistOnce = sync.Once{} ssrfWhitelist = nil ssrfWhitelistAtomic.Store(nil) + invalidateSSRFOutboundValidationCache() } // FormatSSRFError takes the error returned by ValidateURLForSSRF and wraps @@ -1242,6 +1293,13 @@ func ValidateURLForSSRF(rawURL string) error { return fmt.Errorf("URL has no hostname") } + // A whitelist relaxes host/IP restrictions only. It must never turn other + // schemes (file://, gopher://, etc.) into valid outbound request targets. + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("invalid scheme: %s (only http/https allowed)", scheme) + } + // If the host is whitelisted, skip the heavy checks. if IsSSRFWhitelisted(hostname) { return nil diff --git a/internal/utils/security_test.go b/internal/utils/security_test.go index 22fba126..30e55628 100644 --- a/internal/utils/security_test.go +++ b/internal/utils/security_test.go @@ -191,6 +191,12 @@ func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) { rawURL: "https://8.8.8.8/dns-query", wantErr: true, }, + { + name: "whitelist does not allow non-http scheme", + whitelist: "example.com", + rawURL: "gopher://example.com/_stats", + wantErr: true, + }, } for _, tt := range tests { diff --git a/internal/utils/security_transport_test.go b/internal/utils/security_transport_test.go index 296e2417..303df5ae 100644 --- a/internal/utils/security_transport_test.go +++ b/internal/utils/security_transport_test.go @@ -1,11 +1,46 @@ package utils import ( + "context" + "fmt" "net/http" + "strings" "testing" "time" ) +type recordingRoundTripper struct { + called bool +} + +func (r *recordingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + r.called = true + return nil, fmt.Errorf("unexpected network call") +} + +func TestSSRFSafeClientValidatesInitialRequestAtFinalSink(t *testing.T) { + base := &recordingRoundTripper{} + client := NewSSRFSafeHTTPClientWithTransport(DefaultSSRFSafeHTTPClientConfig(), base) + req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data", nil) + if err != nil { + t.Fatal(err) + } + _, err = client.Do(req) + if err == nil || !strings.Contains(err.Error(), "SSRF") { + t.Fatalf("expected final-sink SSRF rejection, got %v", err) + } + if base.called { + t.Fatal("unsafe request reached the base transport") + } +} + +func TestSSRFSafeDialContextRejectsRestrictedPortAtFinalSink(t *testing.T) { + _, err := SSRFSafeDialContext(context.Background(), "tcp", "example.com:6379") + if err == nil || !strings.Contains(err.Error(), "port 6379") { + t.Fatalf("expected restricted-port error, got %v", err) + } +} + // TestNewSSRFSafeTransport_SharedAcrossClients verifies that a single transport // can back multiple clients (global connection pooling) while each client keeps // its own timeout and a redirect policy. @@ -22,11 +57,16 @@ func TestNewSSRFSafeTransport_SharedAcrossClients(t *testing.T) { if first == second { t.Fatal("expected distinct HTTP clients") } - if first.Transport != second.Transport { - t.Fatal("expected clients to share the same transport") + firstGuard, ok := first.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", first.Transport) } - if first.Transport != http.RoundTripper(shared) { - t.Fatal("expected client to use the supplied shared transport") + secondGuard, ok := second.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", second.Transport) + } + if firstGuard.Base != secondGuard.Base || firstGuard.Base != http.RoundTripper(shared) { + t.Fatal("expected clients to share the supplied base transport") } if first.Timeout != 15*time.Second { t.Fatalf("unexpected first timeout: got %v, want %v", first.Timeout, 15*time.Second) @@ -46,8 +86,12 @@ func TestNewSSRFSafeHTTPClient_HasDedicatedTransport(t *testing.T) { if client.Transport == nil { t.Fatal("expected a transport to be set") } - if _, ok := client.Transport.(*http.Transport); !ok { - t.Fatalf("expected *http.Transport, got %T", client.Transport) + guard, ok := client.Transport.(*SSRFValidatingRoundTripper) + if !ok { + t.Fatalf("expected SSRF-validating wrapper, got %T", client.Transport) + } + if _, ok := guard.Base.(*http.Transport); !ok { + t.Fatalf("expected *http.Transport base, got %T", guard.Base) } if client.CheckRedirect == nil { t.Fatal("expected SSRF redirect policy to be set") diff --git a/internal/utils/ssrf_outbound_cache.go b/internal/utils/ssrf_outbound_cache.go new file mode 100644 index 00000000..d7ffdbba --- /dev/null +++ b/internal/utils/ssrf_outbound_cache.go @@ -0,0 +1,135 @@ +package utils + +import ( + "fmt" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/singleflight" +) + +// outboundSSRFValidationTTL bounds how long a successful RoundTrip/redirect +// validation for the same origin may be reused. DNS rebinding at the TCP sink +// is still blocked by SSRFSafeDialContext on every new connection. +const outboundSSRFValidationTTL = 60 * time.Second + +var ( + ssrfOutboundCacheGen atomic.Uint64 + ssrfOutboundCache sync.Map // string -> *ssrfOutboundCacheEntry + ssrfOutboundValidateGroup singleflight.Group + ssrfOutboundValidateMisses atomic.Uint64 // test-only counter +) + +type ssrfOutboundCacheEntry struct { + err error + expiresAt time.Time +} + +// validateURLForSSRFForOutbound validates URLs on the hot outbound path +// (RoundTripper, redirect checks). Results are cached per origin for a short +// TTL so high-frequency clients do not repeat DNS lookups on every request. +// Handler/input boundaries should keep calling ValidateURLForSSRF directly. +func validateURLForSSRFForOutbound(rawURL string) error { + if rawURL == "" { + return nil + } + + cacheKey, ok := outboundSSRFCacheKey(rawURL) + if !ok { + return ValidateURLForSSRF(rawURL) + } + + now := time.Now() + if cached, ok := ssrfOutboundCache.Load(cacheKey); ok { + entry := cached.(*ssrfOutboundCacheEntry) + if now.Before(entry.expiresAt) { + return entry.err + } + ssrfOutboundCache.Delete(cacheKey) + } + + result, err, _ := ssrfOutboundValidateGroup.Do(cacheKey, func() (any, error) { + if cached, ok := ssrfOutboundCache.Load(cacheKey); ok { + entry := cached.(*ssrfOutboundCacheEntry) + if time.Now().Before(entry.expiresAt) { + return entry.err, entry.err + } + } + + ssrfOutboundValidateMisses.Add(1) + validationErr := ValidateURLForSSRF(rawURL) + ssrfOutboundCache.Store(cacheKey, &ssrfOutboundCacheEntry{ + err: validationErr, + expiresAt: time.Now().Add(outboundSSRFValidationTTL), + }) + return validationErr, validationErr + }) + if err != nil { + return err + } + if validationErr, ok := result.(error); ok { + return validationErr + } + return nil +} + +func outboundSSRFCacheKey(rawURL string) (string, bool) { + normalized := rawURL + if !strings.Contains(normalized, "://") { + normalized = "https://" + normalized + } + parsed, err := url.Parse(normalized) + if err != nil { + return "", false + } + + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "", false + } + + host := strings.ToLower(parsed.Hostname()) + if host == "" { + return "", false + } + + port := parsed.Port() + if port == "" { + if scheme == "https" { + port = "443" + } else { + port = "80" + } + } + + var origin string + if strings.Contains(host, ":") { + origin = fmt.Sprintf("%s://[%s]:%s", scheme, host, port) + } else { + origin = fmt.Sprintf("%s://%s:%s", scheme, host, port) + } + + return fmt.Sprintf("%d|%s", ssrfOutboundCacheGen.Load(), origin), true +} + +func invalidateSSRFOutboundValidationCache() { + ssrfOutboundCacheGen.Add(1) + ssrfOutboundCache = sync.Map{} + ssrfOutboundValidateGroup = singleflight.Group{} +} + +// ResetSSRFOutboundValidationCacheForTest clears the outbound validation cache. +// NOT for production use. +func ResetSSRFOutboundValidationCacheForTest() { + invalidateSSRFOutboundValidationCache() + ssrfOutboundValidateMisses.Store(0) +} + +// SSRFOutboundValidationMissesForTest returns how many uncached outbound +// validations have run since the last ResetSSRFOutboundValidationCacheForTest. +func SSRFOutboundValidationMissesForTest() uint64 { + return ssrfOutboundValidateMisses.Load() +} diff --git a/internal/utils/ssrf_outbound_cache_test.go b/internal/utils/ssrf_outbound_cache_test.go new file mode 100644 index 00000000..b3856478 --- /dev/null +++ b/internal/utils/ssrf_outbound_cache_test.go @@ -0,0 +1,101 @@ +package utils + +import ( + "net/http" + "testing" + "time" +) + +func TestOutboundSSRFValidationCacheReusesSameOrigin(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/a"); err != nil { + t.Fatalf("first validation failed: %v", err) + } + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/b"); err != nil { + t.Fatalf("second validation failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 1 { + t.Fatalf("expected 1 cache miss for same origin, got %d", got) + } +} + +func TestOutboundSSRFValidationCacheInvalidatesOnWhitelistChange(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil { + t.Fatalf("initial validation failed: %v", err) + } + if SSRFOutboundValidationMissesForTest() != 1 { + t.Fatalf("expected one miss before whitelist change, got %d", SSRFOutboundValidationMissesForTest()) + } + + SetSSRFWhitelistFromRaw("cache-test.example.com,other.example.com") + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil { + t.Fatalf("validation after whitelist change failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 2 { + t.Fatalf("expected cache miss after whitelist bump, got %d misses", got) + } +} + +func TestOutboundSSRFValidationCacheExpires(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil { + t.Fatalf("initial validation failed: %v", err) + } + + cacheKey, ok := outboundSSRFCacheKey("https://cache-test.example.com/a") + if !ok { + t.Fatal("expected cache key") + } + raw, ok := ssrfOutboundCache.Load(cacheKey) + if !ok { + t.Fatal("expected cached entry") + } + entry := raw.(*ssrfOutboundCacheEntry) + entry.expiresAt = time.Now().Add(-time.Second) + ssrfOutboundCache.Store(cacheKey, entry) + + if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil { + t.Fatalf("validation after expiry failed: %v", err) + } + if got := SSRFOutboundValidationMissesForTest(); got != 2 { + t.Fatalf("expected revalidation after TTL expiry, got %d misses", got) + } +} + +func TestSSRFValidatingRoundTripperUsesOutboundCache(t *testing.T) { + ResetSSRFWhitelistForTest() + ResetSSRFOutboundValidationCacheForTest() + SetSSRFWhitelistFromRaw("cache-test.example.com") + + base := &recordingRoundTripper{} + rt := &SSRFValidatingRoundTripper{Base: base} + + req1, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/a", nil) + if err != nil { + t.Fatal(err) + } + req2, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/b", nil) + if err != nil { + t.Fatal(err) + } + + if _, err := rt.RoundTrip(req1); err == nil { + t.Fatal("expected base transport error, validation should pass") + } + if _, err := rt.RoundTrip(req2); err == nil { + t.Fatal("expected base transport error, validation should pass") + } + if got := SSRFOutboundValidationMissesForTest(); got != 1 { + t.Fatalf("round tripper expected one outbound validation miss, got %d", got) + } +}