fix(security): harden outbound URL SSRF protection (#2626)
* fix(security): harden outbound URL SSRF protection * test(security): add SSRF validation tests for outbound URL caching Introduce tests for outbound SSRF validation cache to ensure proper handling of URL validation, including cache reuse for the same origin, cache invalidation on whitelist changes, and expiration of cached entries. This enhances the robustness of SSRF protection mechanisms in the application. * refactor: clean up whitespace in engine_factory.go and ssrf_outbound_cache.go Removed unnecessary blank lines in engine_factory.go and adjusted variable declarations for consistency in ssrf_outbound_cache.go. This improves code readability without altering functionality.main
parent
090612da54
commit
1030832c4b
|
|
@ -717,10 +717,10 @@ LANGFUSE_HOST=http://langfuse-web:3000
|
|||
#
|
||||
# ===== 向量库地址的 SSRF 校验 =====
|
||||
# 通过 API 创建/测试向量库连接时,地址会经过 SSRF 校验。docker-compose 内置服务
|
||||
# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。
|
||||
# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng, minio)已通过 SSRF_WHITELIST_EXTRA 默认放行。
|
||||
# 如使用外部私网地址或被拦截端口(5432, 9200 等),请将主机加入 SSRF_WHITELIST_EXTRA
|
||||
# (注意:自定义该变量会覆盖 compose 默认值,需重新包含内置服务名),例如:
|
||||
# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,my-opensearch.internal
|
||||
# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio,my-opensearch.internal
|
||||
#
|
||||
# Doris Stream Load:FE 会将请求 307 重定向到 BE;docker compose 默认放行
|
||||
# doris-fe 与 doris-be(见 SSRF_WHITELIST_EXTRA)。
|
||||
|
|
|
|||
|
|
@ -285,17 +285,17 @@ services:
|
|||
- SSRF_WHITELIST=${SSRF_WHITELIST:-}
|
||||
# 保留原始 URL 的图片域名白名单(逗号分隔,不替换为 provider://)
|
||||
- IMAGE_HOST_KEEP_URL=${IMAGE_HOST_KEEP_URL:-}
|
||||
# Always allow the optional searxng sidecar and the bundled vector-store
|
||||
# service hostnames (compose service names). Vector-store create/test now
|
||||
# runs SSRF validation on user-supplied addresses, which would otherwise
|
||||
# reject these private docker-network hosts; whitelisting them keeps the
|
||||
# Always allow the optional searxng sidecar, bundled vector stores, and
|
||||
# the default MinIO storage host (compose service names). User-controlled
|
||||
# endpoints are SSRF-validated and these private docker-network hosts
|
||||
# would otherwise be rejected; whitelisting them keeps the
|
||||
# out-of-box experience. Merged on top of SSRF_WHITELIST so user
|
||||
# overrides don't clobber it. (postgres is intentionally NOT listed: it
|
||||
# is not a user-registerable vector store, and whitelisting the app's own
|
||||
# DB host would skip the 5432 port block.)
|
||||
# doris-be: Stream Load follows FE→BE redirects that must receive Basic
|
||||
# credentials on whitelisted BE hosts.
|
||||
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be}
|
||||
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be,minio}
|
||||
- CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5}
|
||||
- JWT_SECRET=${JWT_SECRET:-}
|
||||
# File size limit (in MB)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from docreader.config import CONFIG
|
|||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
from docreader.utils.ssrf import is_ssrf_safe_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -34,6 +35,13 @@ _MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
|||
_IMAGE_FILE_NUM_RE = re.compile(r"^imageFile(\d+)\.", re.I)
|
||||
|
||||
|
||||
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
"""Health probes never need redirects; refusing them closes redirect SSRF."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def _override_str(overrides: Optional[Mapping[str, Any]], key: str, default: str = "") -> str:
|
||||
if overrides:
|
||||
v = overrides.get(key)
|
||||
|
|
@ -75,11 +83,16 @@ def _ping_hybrid(
|
|||
|
||||
base = url.rstrip("/")
|
||||
health_url = f"{base}/health"
|
||||
safe, reason = is_ssrf_safe_url(health_url)
|
||||
if not safe:
|
||||
return False, f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}"
|
||||
|
||||
opener = urllib.request.build_opener(_NoRedirectHandler())
|
||||
last_err = ""
|
||||
for attempt in range(max(1, retries)):
|
||||
try:
|
||||
req = urllib.request.Request(health_url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=timeout_sec) as resp:
|
||||
with opener.open(req, timeout=timeout_sec) as resp:
|
||||
if 200 <= resp.status < 300:
|
||||
return True, ""
|
||||
last_err = f"hybrid 健康检查 HTTP {resp.status}: {health_url}"
|
||||
|
|
@ -278,6 +291,11 @@ def _run_convert(
|
|||
kwargs["hybrid"] = hybrid
|
||||
hybrid_url = _resolve_hybrid_url(overrides)
|
||||
if hybrid_url:
|
||||
safe, reason = is_ssrf_safe_url(hybrid_url)
|
||||
if not safe:
|
||||
raise RuntimeError(
|
||||
f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}"
|
||||
)
|
||||
kwargs["hybrid_url"] = hybrid_url
|
||||
hybrid_mode = _override_str(overrides, "odl_hybrid_mode", CONFIG.odl_hybrid_mode)
|
||||
if hybrid_mode:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Unit tests for OpenDataLoader parser helpers (no JVM required)."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
|
@ -10,12 +11,41 @@ from docreader.parser.opendataloader_parser import (
|
|||
_collect_images_under_output,
|
||||
_find_markdown_file,
|
||||
_normalize_odl_image_url,
|
||||
_ping_hybrid,
|
||||
_run_convert,
|
||||
_rewrite_markdown_image_refs,
|
||||
opendataloader_available,
|
||||
)
|
||||
|
||||
|
||||
class OpenDataLoaderHelpersTest(unittest.TestCase):
|
||||
def test_hybrid_health_probe_blocks_private_url_before_request(self):
|
||||
with mock.patch(
|
||||
"docreader.parser.opendataloader_parser.is_ssrf_safe_url",
|
||||
return_value=(False, "restricted test address"),
|
||||
), mock.patch("urllib.request.build_opener") as build_opener:
|
||||
ok, msg = _ping_hybrid("http://127.0.0.1:8080", retries=1)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("SSRF", msg)
|
||||
build_opener.assert_not_called()
|
||||
|
||||
def test_convert_blocks_private_hybrid_url_at_final_sink(self):
|
||||
fake_module = mock.Mock()
|
||||
with mock.patch.dict(sys.modules, {"opendataloader_pdf": fake_module}):
|
||||
with self.assertRaisesRegex(RuntimeError, "SSRF"):
|
||||
_run_convert(
|
||||
"/tmp/input.pdf",
|
||||
"/tmp/output",
|
||||
"/tmp/output/images",
|
||||
overrides={
|
||||
"odl_hybrid": "docling-fast",
|
||||
"odl_hybrid_url": "http://169.254.169.254/latest/meta-data",
|
||||
},
|
||||
)
|
||||
|
||||
fake_module.convert.assert_not_called()
|
||||
|
||||
def test_find_markdown_prefers_stem_match(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
other = os.path.join(d, "other.md")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import ipaddress
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
|
@ -36,6 +37,20 @@ class TestSSRFValidation(unittest.TestCase):
|
|||
self.assertFalse(safe)
|
||||
self.assertTrue(reason)
|
||||
|
||||
def test_blocks_invalid_port(self):
|
||||
safe, reason = is_ssrf_safe_url("https://example.com:99999/path")
|
||||
self.assertFalse(safe)
|
||||
self.assertIn("invalid port", reason)
|
||||
|
||||
def test_blocks_ipv4_mapped_private_ipv6_resolution(self):
|
||||
with patch(
|
||||
"docreader.utils.ssrf._resolve_host_ips",
|
||||
return_value=((ipaddress.ip_address("::ffff:127.0.0.1"),), None),
|
||||
):
|
||||
safe, reason = is_ssrf_safe_url("https://example.invalid/path")
|
||||
self.assertFalse(safe)
|
||||
self.assertIn("restricted", reason)
|
||||
|
||||
def test_allows_public_https(self):
|
||||
safe, reason = is_ssrf_safe_url("https://example.com/article")
|
||||
self.assertTrue(safe, reason)
|
||||
|
|
|
|||
|
|
@ -161,6 +161,21 @@ def _is_restricted_ip(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) -
|
|||
if isinstance(net, ipaddress.IPv4Network) and ip in net:
|
||||
return f"restricted range {net}"
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
embedded_ipv4 = ip.ipv4_mapped
|
||||
if embedded_ipv4 is not None:
|
||||
reason = _is_restricted_ip(embedded_ipv4)
|
||||
if reason:
|
||||
return f"IPv4-mapped {reason}"
|
||||
if ip.sixtofour is not None:
|
||||
reason = _is_restricted_ip(ip.sixtofour)
|
||||
if reason:
|
||||
return f"6to4-embedded {reason}"
|
||||
if ip.teredo is not None:
|
||||
server_ip, client_ip = ip.teredo
|
||||
for label, embedded_ip in (("server", server_ip), ("client", client_ip)):
|
||||
reason = _is_restricted_ip(embedded_ip)
|
||||
if reason:
|
||||
return f"Teredo {label} embeds {reason}"
|
||||
# Site-local (fec0::/10)
|
||||
if (ip.packed[0] == 0xFE) and (ip.packed[1] & 0xC0) == 0xC0:
|
||||
return "site-local IPv6 address"
|
||||
|
|
@ -238,7 +253,10 @@ def is_ssrf_safe_url(raw_url: str) -> Tuple[bool, str]:
|
|||
f"hostname {hostname_lower} resolves to restricted IP {resolved_ip}: {reason}",
|
||||
)
|
||||
|
||||
port = parsed.port
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
return False, f"invalid port: {exc}"
|
||||
if port is not None and str(port) in BLOCKED_PORTS:
|
||||
return False, f"port {port} is blocked for security reasons"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
package opensearch
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// End-to-end repository tests intentionally use loopback httptest servers.
|
||||
secutils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost,opensearch.example.com")
|
||||
code := m.Run()
|
||||
secutils.SetSSRFWhitelistFromRaw("")
|
||||
os.Exit(code)
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// NewOpenSearchClient builds a TLS-hardened, pool-tuned *osapi.Client for
|
||||
|
|
@ -35,13 +36,16 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) {
|
|||
if cfg == nil || cfg.Addr == "" {
|
||||
return nil, fmt.Errorf("opensearch: ConnectionConfig.Addr required: %w", ErrConfigInvalid)
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(cfg.Addr); err != nil {
|
||||
return nil, fmt.Errorf("opensearch: address failed SSRF validation: %w", err)
|
||||
}
|
||||
transport := buildHTTPTransport(cfg.InsecureSkipVerify)
|
||||
return osapi.NewClient(osapi.Config{
|
||||
Client: opensearch.Config{
|
||||
Addresses: []string{cfg.Addr},
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
Transport: transport,
|
||||
Transport: &secutils.SSRFValidatingRoundTripper{Base: transport},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -54,6 +58,8 @@ func NewOpenSearchClient(cfg *types.ConnectionConfig) (*osapi.Client, error) {
|
|||
// final client fragile across SDK versions).
|
||||
func buildHTTPTransport(insecureSkipVerify bool) *http.Transport {
|
||||
return &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: secutils.SSRFSafeDialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: insecureSkipVerify, //nolint:gosec — operator opt-in flag
|
||||
|
|
|
|||
|
|
@ -32,6 +32,22 @@ type cosFileService struct {
|
|||
|
||||
const cosScheme = "cos://"
|
||||
|
||||
func newCOSHTTPClient(secretID, secretKey string) *http.Client {
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client := utils.NewSSRFSafeHTTPClient(httpConfig)
|
||||
// COS GetCredential expects AuthorizationTransport to remain the outermost
|
||||
// transport. Put the per-request SSRF guard immediately underneath it while
|
||||
// retaining the safe client's redirect policy.
|
||||
client.Transport = &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
},
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// newCosClient creates a bare cosFileService with just the SDK client initialised.
|
||||
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
|
||||
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
|
||||
|
|
@ -41,12 +57,7 @@ func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileServi
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, newCOSHTTPClient(secretID, secretKey))
|
||||
return &cosFileService{client: client, bucketURL: bucketURL, bucketName: bucketName, region: region}, nil
|
||||
}
|
||||
|
||||
|
|
@ -72,12 +83,7 @@ func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, co
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, newCOSHTTPClient(secretId, secretKey))
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStorageClientsRejectUnsafeEndpointAtConstruction(t *testing.T) {
|
||||
const endpoint = "http://169.254.169.254/latest/meta-data"
|
||||
tests := map[string]func() error{
|
||||
"minio": func() error {
|
||||
_, err := newMinioClient(endpoint, "ak", "sk", "bucket", false)
|
||||
return err
|
||||
},
|
||||
"s3": func() error {
|
||||
_, err := newS3Client(endpoint, "ak", "sk", "bucket", "region", "", true)
|
||||
return err
|
||||
},
|
||||
"obs": func() error {
|
||||
return CheckObsConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket")
|
||||
},
|
||||
"oss": func() error {
|
||||
_, err := newOSSClient(endpoint, "region", "ak", "sk")
|
||||
return err
|
||||
},
|
||||
"tos": func() error {
|
||||
return CheckTosConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket")
|
||||
},
|
||||
"ks3": func() error {
|
||||
_, err := newKS3Client(endpoint, "region", "ak", "sk")
|
||||
return err
|
||||
},
|
||||
}
|
||||
for name, run := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("expected unsafe endpoint to be rejected")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -53,6 +53,9 @@ func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathP
|
|||
}
|
||||
|
||||
func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe KS3 endpoint: %w", err)
|
||||
}
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
client := ks3s3.New(&ks3aws.Config{
|
||||
Credentials: creds,
|
||||
|
|
@ -62,6 +65,7 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err
|
|||
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
|
||||
SignerVersion: "V2", // KS3 recommends V2 signing
|
||||
MaxRetries: 3,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
return client, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,9 +28,16 @@ type minioFileService struct {
|
|||
// Shared by NewMinioFileService (which also ensures the bucket exists) and
|
||||
// CheckMinioConnectivity (read-only probe).
|
||||
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe MinIO endpoint: %w", err)
|
||||
}
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
|
@ -43,12 +44,16 @@ func NewObsFileService(
|
|||
endpoint, region, accessKeyID, secretAccessKey, bucketName string,
|
||||
pathPrefix string,
|
||||
) (interfaces.FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe OBS endpoint: %w", err)
|
||||
}
|
||||
|
||||
client := s3.New(s3.Options{
|
||||
Region: region,
|
||||
EndpointResolver: &obsEndpointResolver{url: endpoint},
|
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""),
|
||||
UsePathStyle: true,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
|
||||
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
|
||||
|
|
@ -76,6 +81,9 @@ func NewObsFileService(
|
|||
}
|
||||
|
||||
func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("unsafe OBS endpoint: %w", err)
|
||||
}
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
|
|
@ -84,6 +92,7 @@ func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secr
|
|||
EndpointResolver: &obsEndpointResolver{url: endpoint},
|
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
UsePathStyle: true,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
|
||||
_, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{
|
||||
|
|
|
|||
|
|
@ -34,12 +34,16 @@ const ossScheme = "oss://"
|
|||
|
||||
// newOSSClient creates an OSS client using the official Aliyun SDK v2.
|
||||
func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe OSS endpoint: %w", err)
|
||||
}
|
||||
creds := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
|
||||
|
||||
cfg := oss.LoadDefaultConfig().
|
||||
WithCredentialsProvider(creds).
|
||||
WithRegion(region).
|
||||
WithEndpoint(endpoint)
|
||||
WithEndpoint(endpoint).
|
||||
WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()))
|
||||
|
||||
return oss.NewClient(cfg), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ func TestNewOSSClient(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "custom endpoint",
|
||||
endpoint: "https://custom-oss-endpoint.com",
|
||||
endpoint: "https://example.com",
|
||||
region: "cn-shanghai",
|
||||
accessKey: "ak",
|
||||
secretKey: "sk",
|
||||
|
|
@ -145,6 +145,12 @@ func TestNewOSSClient(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestNewOSSClientRejectsUnsafeEndpoint(t *testing.T) {
|
||||
if _, err := newOSSClient("http://127.0.0.1:9000", "cn-hangzhou", "ak", "sk"); err == nil {
|
||||
t.Fatal("expected loopback OSS endpoint to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ type s3FileService struct {
|
|||
|
||||
// newS3Client creates a bare s3FileService with just the SDK client initialised.
|
||||
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe S3 endpoint: %w", err)
|
||||
}
|
||||
var cfg aws.Config
|
||||
var err error
|
||||
|
||||
|
|
@ -55,16 +58,20 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
|
|||
// Create S3 client with custom endpoint if provided.
|
||||
// For S3-compatible services (non-AWS), use path-style addressing
|
||||
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
|
||||
httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())
|
||||
var client *s3.Client
|
||||
if endpoint != "" {
|
||||
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = usePathStyle
|
||||
o.HTTPClient = httpClient
|
||||
})
|
||||
} else {
|
||||
// Standard AWS S3
|
||||
client = s3.NewFromConfig(cfg)
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.HTTPClient = httpClient
|
||||
})
|
||||
}
|
||||
|
||||
// Normalize pathPrefix: ensure it ends with '/' if not empty
|
||||
|
|
|
|||
|
|
@ -36,10 +36,17 @@ func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathP
|
|||
|
||||
// NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket.
|
||||
func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe TOS endpoint: %w", err)
|
||||
}
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS client: %w", err)
|
||||
|
|
@ -58,6 +65,9 @@ func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, buc
|
|||
endpoint,
|
||||
tos.WithRegion(tempRegion),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err)
|
||||
|
|
@ -87,10 +97,16 @@ func (s *tosFileService) CheckConnectivity(ctx context.Context) error {
|
|||
|
||||
// CheckTosConnectivity tests TOS connectivity using the provided credentials.
|
||||
func CheckTosConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("unsafe TOS endpoint: %w", err)
|
||||
}
|
||||
client, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize TOS client: %w", err)
|
||||
|
|
|
|||
|
|
@ -3593,6 +3593,16 @@ func (s *knowledgeService) convert(
|
|||
}
|
||||
mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides)
|
||||
applyParserRuleOverrides(mergedOverrides, eff.ChunkingConfig, fileType)
|
||||
if err := validateParserEngineOverrideURLs(mergedOverrides); err != nil {
|
||||
logger.Errorf(ctx, "Parser endpoint rejected for SSRF protection: %v", err)
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "Parser endpoint is not allowed for security reasons"
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
s.failStage(ctx, knowledge.ID, types.StageDocReader,
|
||||
werrors.ErrCodeDocReaderParseFailed, knowledge.ErrorMessage, err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if isURL {
|
||||
if err := secutils.ValidateURLForSSRF(payload.URL); err != nil {
|
||||
|
|
|
|||
|
|
@ -33,7 +33,9 @@ func TestDownloadFileFromURLBlocksRedirectToLoopback(t *testing.T) {
|
|||
if err == nil {
|
||||
t.Fatal("expected redirect to loopback to be blocked")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "redirect blocked") && !strings.Contains(err.Error(), "connection blocked") {
|
||||
if !strings.Contains(err.Error(), "redirect blocked") &&
|
||||
!strings.Contains(err.Error(), "connection blocked") &&
|
||||
!strings.Contains(err.Error(), "outbound request blocked") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@ func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types
|
|||
if service.TransportType == types.MCPTransportStdio {
|
||||
return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead")
|
||||
}
|
||||
if err := mcp.ValidateServiceOutboundURLs(service); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set default advanced config if not provided
|
||||
if service.AdvancedConfig == nil {
|
||||
|
|
@ -248,6 +251,9 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
|
|||
}
|
||||
|
||||
// Update timestamp
|
||||
if err := mcp.ValidateServiceOutboundURLs(existing); err != nil {
|
||||
return err
|
||||
}
|
||||
existing.UpdatedAt = time.Now()
|
||||
|
||||
if err := s.mcpServiceRepo.Update(ctx, existing); err != nil {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
var parserOutboundURLKeys = []string{
|
||||
"mineru_endpoint",
|
||||
"mineru_vlm_server_url",
|
||||
"odl_hybrid_url",
|
||||
"paddleocr_vl_endpoint",
|
||||
"paddleocr_vl_cloud_base_url",
|
||||
}
|
||||
|
||||
// validateParserEngineOverrideURLs validates every parser override that can
|
||||
// cause this process or the trusted DocReader service to make an outbound
|
||||
// request. Per-upload overrides are included because API callers can provide
|
||||
// the generic parser_engine_overrides map directly.
|
||||
func validateParserEngineOverrideURLs(overrides map[string]string) error {
|
||||
for _, key := range parserOutboundURLKeys {
|
||||
rawURL := strings.TrimSpace(overrides[key])
|
||||
if rawURL == "" {
|
||||
continue
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(rawURL); err != nil {
|
||||
return fmt.Errorf("%s failed SSRF validation: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package service
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidateParserOverrideURLsRejectsEverySupportedURLKey(t *testing.T) {
|
||||
for _, key := range parserOutboundURLKeys {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
err := validateParserEngineOverrideURLs(map[string]string{
|
||||
key: "http://169.254.169.254/latest/meta-data",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected %s to be rejected", key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateParserOverrideURLsIgnoresNonURLKeys(t *testing.T) {
|
||||
if err := validateParserEngineOverrideURLs(map[string]string{
|
||||
"mineru_model_version": "pipeline",
|
||||
}); err != nil {
|
||||
t.Fatalf("unexpected error for non-URL override: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -15,6 +14,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/go-sql-driver/mysql" // MySQL driver for database/sql, used by Doris connection test
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"github.com/weaviate/weaviate-go-client/v5/weaviate"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate/auth"
|
||||
wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
const connectionTestTimeout = 10 * time.Second
|
||||
|
|
@ -71,14 +72,14 @@ func testElasticsearchConnection(ctx context.Context, config types.ConnectionCon
|
|||
req.SetBasicAuth(config.Username, config.Password)
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: connectionTestTimeout,
|
||||
clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
clientCfg.Timeout = connectionTestTimeout
|
||||
client := secutils.NewSSRFSafeHTTPClient(clientCfg)
|
||||
client.CheckRedirect = func(*http.Request, []*http.Request) error {
|
||||
// A health probe must not follow redirects: a malicious or compromised
|
||||
// endpoint could return a 302 to an internal address, defeating the
|
||||
// SSRF check that only validated the original (user-supplied) Addr.
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -155,6 +156,9 @@ func testQdrantConnection(ctx context.Context, config types.ConnectionConfig) (s
|
|||
Port: port,
|
||||
APIKey: config.APIKey,
|
||||
UseTLS: config.UseTLS,
|
||||
GrpcOptions: []grpc.DialOption{
|
||||
grpc.WithContextDialer(secutils.SSRFSafeGRPCDialer),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", errors.NewBadRequestError("failed to create qdrant client: invalid configuration")
|
||||
|
|
@ -184,7 +188,7 @@ func testMilvusConnection(ctx context.Context, config types.ConnectionConfig) (s
|
|||
addr = "localhost:19530"
|
||||
}
|
||||
|
||||
conn, err := (&net.Dialer{}).DialContext(testCtx, "tcp", addr)
|
||||
conn, err := secutils.SSRFSafeDialContext(testCtx, "tcp", addr)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Milvus connection test failed: %v", err)
|
||||
return "", errors.NewBadRequestError("failed to connect to milvus: connection refused or server unreachable")
|
||||
|
|
@ -201,6 +205,9 @@ func testTencentVectorDBConnection(ctx context.Context, config types.ConnectionC
|
|||
client, err := tcvectordb.NewRpcClient(config.Addr, config.Username, config.APIKey, &tcvectordb.ClientOption{
|
||||
ReadConsistency: tcvectordb.EventualConsistency,
|
||||
Timeout: connectionTestTimeout,
|
||||
Transport: &secutils.SSRFValidatingRoundTripper{
|
||||
Base: secutils.NewSSRFSafeTransport(secutils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Tencent VectorDB connection test failed: %v", err)
|
||||
|
|
@ -233,7 +240,8 @@ func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig)
|
|||
}
|
||||
|
||||
weaviateCfg := weaviate.Config{
|
||||
Host: host,
|
||||
Host: host,
|
||||
ConnectionClient: secutils.NewSSRFSafeHTTPClient(secutils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
GrpcConfig: &wgrpc.Config{
|
||||
Host: grpcAddress,
|
||||
},
|
||||
|
|
@ -291,7 +299,8 @@ func testDorisConnection(ctx context.Context, config types.ConnectionConfig) (st
|
|||
cfg := mysql.NewConfig()
|
||||
cfg.User = config.Username
|
||||
cfg.Passwd = config.Password
|
||||
cfg.Net = "tcp"
|
||||
secutils.RegisterMySQLSSRFDialer()
|
||||
cfg.Net = secutils.MySQLSSRFNetwork
|
||||
cfg.Addr = config.Addr
|
||||
cfg.DBName = database
|
||||
cfg.Timeout = 5 * time.Second
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestTestConnection_MilvusBlocksUnsafeAddrAtDialSink(t *testing.T) {
|
||||
svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := svc.TestConnection(ctx, types.MilvusRetrieverEngineType, types.ConnectionConfig{
|
||||
Addr: "169.254.169.254:19530",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected milvus connectivity probe to reject unsafe address")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "failed to connect to milvus") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestConnection_DorisBlocksUnsafeAddrAtDialSink(t *testing.T) {
|
||||
svc := NewVectorStoreService(&mockVectorStoreRepo{}, nil, nil, nil, nil)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := svc.TestConnection(ctx, types.DorisRetrieverEngineType, types.ConnectionConfig{
|
||||
Addr: "169.254.169.254:9030",
|
||||
Database: "weknora",
|
||||
Username: "root",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected doris connectivity probe to reject unsafe address")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "failed to connect to doris") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +72,9 @@ func (s *weKnoraCloudService) verifyCredentials(ctx context.Context, appID, appS
|
|||
logger.Infof(ctx, "credential verification request: method=GET url=%s app_id=%s request_id=%s ",
|
||||
healthURL, appID, requestID)
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
clientCfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
clientCfg.Timeout = 10 * time.Second
|
||||
client := utils.NewSSRFSafeHTTPClient(clientCfg)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "credential verification HTTP failed: url=%s err=%v", healthURL, err)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -33,6 +34,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
|
||||
)
|
||||
|
||||
|
|
@ -58,6 +60,9 @@ func createEngineServiceFromStore(
|
|||
cfg *config.Config,
|
||||
auditSink openSearchRepo.AuditSink,
|
||||
) (interfaces.RetrieveEngineService, error) {
|
||||
if err := validateRuntimeVectorStoreAddresses(store); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch store.EngineType {
|
||||
case types.PostgresRetrieverEngineType:
|
||||
return createPostgresEngine(store, db)
|
||||
|
|
@ -82,6 +87,48 @@ func createEngineServiceFromStore(
|
|||
}
|
||||
}
|
||||
|
||||
// validateRuntimeVectorStoreAddresses is the final guard for persisted or
|
||||
// imported vector-store rows. Create/test handlers validate the same fields at
|
||||
// the input boundary, but runtime construction must not assume every stored row
|
||||
// was written through those handlers.
|
||||
func validateRuntimeVectorStoreAddresses(store types.VectorStore) error {
|
||||
cc := store.ConnectionConfig
|
||||
check := func(label, endpoint string) error {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("%s failed SSRF validation: %w", label, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
switch store.EngineType {
|
||||
case types.PostgresRetrieverEngineType, types.SQLiteRetrieverEngineType:
|
||||
return nil
|
||||
case types.ElasticsearchRetrieverEngineType,
|
||||
types.OpenSearchRetrieverEngineType,
|
||||
types.MilvusRetrieverEngineType,
|
||||
types.TencentVectorDBRetrieverEngineType,
|
||||
types.DorisRetrieverEngineType:
|
||||
return check("vector store address", cc.Addr)
|
||||
case types.QdrantRetrieverEngineType:
|
||||
endpoint := cc.Host
|
||||
if endpoint != "" && cc.Port != 0 {
|
||||
endpoint = net.JoinHostPort(strings.Trim(cc.Host, "[]"), strconv.Itoa(cc.Port))
|
||||
}
|
||||
return check("qdrant address", endpoint)
|
||||
case types.WeaviateRetrieverEngineType:
|
||||
if err := check("weaviate HTTP address", cc.Host); err != nil {
|
||||
return err
|
||||
}
|
||||
return check("weaviate gRPC address", cc.GrpcAddress)
|
||||
default:
|
||||
return fmt.Errorf("vector store engine %q has no SSRF address policy", store.EngineType)
|
||||
}
|
||||
}
|
||||
|
||||
// createOpenSearchEngine builds an OpenSearch k-NN retrieve engine. Mirrors
|
||||
// createElasticsearchV8Engine but uses the driver's TLS-hardened client
|
||||
// constructor and injects the audit sink. NewRepository probes the cluster
|
||||
|
|
@ -142,10 +189,16 @@ func isESv7(version string) bool {
|
|||
|
||||
func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
if err := utils.ValidateURLForSSRF(cc.Addr); err != nil {
|
||||
return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err)
|
||||
}
|
||||
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
|
||||
Addresses: []string{cc.Addr},
|
||||
Username: cc.Username,
|
||||
Password: cc.Password,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create elasticsearch v8 client: %w", err)
|
||||
|
|
@ -156,10 +209,16 @@ func createElasticsearchV8Engine(store types.VectorStore, cfg *config.Config) (i
|
|||
|
||||
func createElasticsearchV7Engine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
if err := utils.ValidateURLForSSRF(cc.Addr); err != nil {
|
||||
return nil, fmt.Errorf("elasticsearch address failed SSRF validation: %w", err)
|
||||
}
|
||||
client, err := esv7.NewClient(esv7.Config{
|
||||
Addresses: []string{cc.Addr},
|
||||
Username: cc.Username,
|
||||
Password: cc.Password,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create elasticsearch v7 client: %w", err)
|
||||
|
|
@ -176,10 +235,11 @@ func createQdrantEngine(store types.VectorStore) (interfaces.RetrieveEngineServi
|
|||
}
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: cc.Host,
|
||||
Port: port,
|
||||
APIKey: cc.APIKey,
|
||||
UseTLS: cc.UseTLS,
|
||||
Host: cc.Host,
|
||||
Port: port,
|
||||
APIKey: cc.APIKey,
|
||||
UseTLS: cc.UseTLS,
|
||||
GrpcOptions: []grpc.DialOption{grpc.WithContextDialer(utils.SSRFSafeGRPCDialer)},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create qdrant client: %w", err)
|
||||
|
|
@ -205,8 +265,11 @@ func buildMilvusClientConfig(cc types.ConnectionConfig) milvusclient.ClientConfi
|
|||
}
|
||||
|
||||
milvusCfg := milvusclient.ClientConfig{
|
||||
Address: addr,
|
||||
DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)},
|
||||
Address: addr,
|
||||
DialOptions: []grpc.DialOption{
|
||||
grpc.WithTimeout(5 * time.Second),
|
||||
grpc.WithContextDialer(utils.SSRFSafeGRPCDialer),
|
||||
},
|
||||
}
|
||||
if cc.Username != "" {
|
||||
milvusCfg.Username = cc.Username
|
||||
|
|
@ -236,7 +299,8 @@ func createWeaviateEngine(store types.VectorStore) (interfaces.RetrieveEngineSer
|
|||
}
|
||||
|
||||
weaviateCfg := weaviate.Config{
|
||||
Host: host,
|
||||
Host: host,
|
||||
ConnectionClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
GrpcConfig: &wgrpc.Config{
|
||||
Host: grpcAddress,
|
||||
},
|
||||
|
|
@ -275,7 +339,8 @@ func createDorisEngine(store types.VectorStore) (interfaces.RetrieveEngineServic
|
|||
mc := mysql.NewConfig()
|
||||
mc.User = cc.Username
|
||||
mc.Passwd = cc.Password
|
||||
mc.Net = "tcp"
|
||||
utils.RegisterMySQLSSRFDialer()
|
||||
mc.Net = utils.MySQLSSRFNetwork
|
||||
mc.Addr = cc.Addr
|
||||
mc.DBName = cc.Database
|
||||
mc.Params = map[string]string{"charset": "utf8mb4"}
|
||||
|
|
@ -314,6 +379,9 @@ func createTencentVectorDBEngine(store types.VectorStore) (interfaces.RetrieveEn
|
|||
client, err := tcvectordb.NewRpcClient(cc.Addr, cc.Username, cc.APIKey, &tcvectordb.ClientOption{
|
||||
ReadConsistency: tcvectordb.EventualConsistency,
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create tencent vectordb client: %w", err)
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ func TestBuildMilvusClientConfig_UsesDatabaseName(t *testing.T) {
|
|||
if cfg.DBName != "regdi_ram_haom1" {
|
||||
t.Fatalf("expected DBName to be propagated, got %q", cfg.DBName)
|
||||
}
|
||||
if len(cfg.DialOptions) != 1 {
|
||||
t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions))
|
||||
if len(cfg.DialOptions) != 2 {
|
||||
t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ func TestBuildMilvusClientConfig_DefaultsAddressWhenMissing(t *testing.T) {
|
|||
if cfg.DBName != "" {
|
||||
t.Fatalf("expected empty DBName by default, got %q", cfg.DBName)
|
||||
}
|
||||
if len(cfg.DialOptions) != 1 {
|
||||
t.Fatalf("expected one dial option, got %d", len(cfg.DialOptions))
|
||||
if len(cfg.DialOptions) != 2 {
|
||||
t.Fatalf("expected timeout and SSRF-safe dial options, got %d", len(cfg.DialOptions))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
package container
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Engine wiring tests intentionally use loopback httptest servers.
|
||||
utils.SetSSRFWhitelistFromRaw("127.0.0.1,::1,localhost")
|
||||
code := m.Run()
|
||||
utils.SetSSRFWhitelistFromRaw("")
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestValidateRuntimeVectorStoreAddressesRejectsUnsafeEndpoints(t *testing.T) {
|
||||
tests := []types.VectorStore{
|
||||
{EngineType: types.QdrantRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "169.254.169.254", Port: 6334}},
|
||||
{EngineType: types.MilvusRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:19530"}},
|
||||
{EngineType: types.WeaviateRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Host: "https://example.com", GrpcAddress: "169.254.169.254:50051"}},
|
||||
{EngineType: types.DorisRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:9030"}},
|
||||
{EngineType: types.TencentVectorDBRetrieverEngineType, ConnectionConfig: types.ConnectionConfig{Addr: "169.254.169.254:80"}},
|
||||
}
|
||||
for _, store := range tests {
|
||||
if err := validateRuntimeVectorStoreAddresses(store); err == nil {
|
||||
t.Fatalf("expected %s unsafe address to be rejected", store.EngineType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRuntimeVectorStoreAddressesAllowsNonNetworkEngines(t *testing.T) {
|
||||
for _, engineType := range []types.RetrieverEngineType{
|
||||
types.PostgresRetrieverEngineType,
|
||||
types.SQLiteRetrieverEngineType,
|
||||
} {
|
||||
if err := validateRuntimeVectorStoreAddresses(types.VectorStore{EngineType: engineType}); err != nil {
|
||||
t.Fatalf("unexpected %s validation error: %v", engineType, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/handler/dto"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
mcpsecurity "github.com/Tencent/WeKnora/internal/mcp"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
|
|
@ -74,6 +75,11 @@ func (h *MCPServiceHandler) CreateMCPService(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
}
|
||||
if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil {
|
||||
logger.Warnf(ctx, "SSRF validation failed for MCP service configuration: %v", err)
|
||||
c.Error(errors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.mcpServiceService.CreateMCPService(ctx, &service); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_name": secutils.SanitizeForLog(service.Name)})
|
||||
|
|
@ -333,6 +339,11 @@ func (h *MCPServiceHandler) UpdateMCPService(c *gin.Context) {
|
|||
service.AdvancedConfig.RetryDelay = int(retryDelay)
|
||||
}
|
||||
}
|
||||
if err := mcpsecurity.ValidateServiceOutboundURLs(&service); err != nil {
|
||||
logger.Warnf(ctx, "SSRF validation failed for MCP service update: %v", err)
|
||||
c.Error(errors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.mcpServiceService.UpdateMCPService(ctx, &service); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
|
||||
|
|
|
|||
|
|
@ -28,8 +28,6 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
)
|
||||
|
||||
|
|
@ -863,59 +861,16 @@ func storageEndpointHost(endpoint string) string {
|
|||
return endpoint
|
||||
}
|
||||
|
||||
// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous
|
||||
// address (cloud metadata, loopback, link-local). Unlike the stricter isSSRFSafeURL,
|
||||
// this allows private IPs since MinIO is commonly deployed on internal networks.
|
||||
// It also respects the SSRF_WHITELIST environment variable for whitelisted hosts.
|
||||
// isBlockedStorageEndpoint keeps the legacy handler contract while delegating
|
||||
// to the same fail-closed SSRF policy used by the storage clients themselves.
|
||||
// Private storage endpoints must be explicitly whitelisted by an operator.
|
||||
func isBlockedStorageEndpoint(endpoint string) (bool, string) {
|
||||
host := storageEndpointHost(endpoint)
|
||||
if host == "" {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
return true, "无效的地址"
|
||||
}
|
||||
|
||||
// Check SSRF whitelist first – whitelisted hosts bypass the block check.
|
||||
if secutils.IsSSRFWhitelisted(host) {
|
||||
return false, ""
|
||||
}
|
||||
|
||||
hostLower := strings.ToLower(host)
|
||||
|
||||
blockedHosts := []string{
|
||||
"metadata.google.internal",
|
||||
"metadata.tencentyun.com",
|
||||
"metadata.aws.internal",
|
||||
}
|
||||
for _, bh := range blockedHosts {
|
||||
if hostLower == bh {
|
||||
return true, "该地址不允许访问"
|
||||
}
|
||||
}
|
||||
|
||||
checkIP := func(ip net.IP) (bool, string) {
|
||||
if ip.IsLoopback() {
|
||||
return true, "不允许访问本地回环地址"
|
||||
}
|
||||
if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
|
||||
return true, "不允许访问链路本地地址"
|
||||
}
|
||||
if ip.IsUnspecified() {
|
||||
return true, "无效的地址"
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return checkIP(ip)
|
||||
}
|
||||
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
return false, ""
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if blocked, reason := checkIP(ip); blocked {
|
||||
return blocked, reason
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return true, secutils.FormatSSRFError("存储 Endpoint", endpoint, err)
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
|
@ -1029,15 +984,7 @@ func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *typ
|
|||
// If bucket does not exist, auto-create it
|
||||
if strings.Contains(errMsg, "does not exist") && cfg.BucketName != "" {
|
||||
logger.Info(ctx, "Storage check: bucket does not exist, attempting auto-creation", "bucket", cfg.BucketName)
|
||||
minioClient, clientErr := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if clientErr != nil {
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to create MinIO client: %s", sanitizeStorageCheckError(clientErr))}})
|
||||
return
|
||||
}
|
||||
if mkErr := minioClient.MakeBucket(ctx, cfg.BucketName, minio.MakeBucketOptions{}); mkErr != nil {
|
||||
if _, mkErr := file.NewMinioFileService(endpoint, accessKeyID, secretAccessKey, cfg.BucketName, cfg.UseSSL); mkErr != nil {
|
||||
logger.Error(ctx, "Storage check: failed to create bucket", "bucket", cfg.BucketName, "error", mkErr)
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Failed to auto-create Bucket '%s': %s", cfg.BucketName, sanitizeStorageCheckError(mkErr))}})
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1737,5 +1737,15 @@ func validateParserEngineOutboundURLs(cfg *types.ParserEngineConfig) error {
|
|||
return fmt.Errorf("mineru_vlm_server_url failed SSRF validation: %v", err)
|
||||
}
|
||||
}
|
||||
if odlURL := strings.TrimSpace(cfg.ODLHybridURL); odlURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(odlURL); err != nil {
|
||||
return fmt.Errorf("odl_hybrid_url failed SSRF validation: %v", err)
|
||||
}
|
||||
}
|
||||
if endpoint := strings.TrimSpace(cfg.PaddleOCRVLEndpoint); endpoint != "" {
|
||||
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("paddleocr_vl_endpoint failed SSRF validation: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,10 @@ import (
|
|||
)
|
||||
|
||||
// httpClient is a shared HTTP client with a reasonable timeout for DingTalk API calls.
|
||||
var httpClient = &http.Client{Timeout: 15 * time.Second}
|
||||
var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 15 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
// apiBaseURL is the DingTalk OpenAPI host. Overridable in tests.
|
||||
var apiBaseURL = "https://api.dingtalk.com"
|
||||
|
|
@ -420,6 +423,9 @@ func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, r
|
|||
}
|
||||
|
||||
func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, content string) error {
|
||||
if err := secutils.ValidateURLForSSRF(webhookURL); err != nil {
|
||||
return fmt.Errorf("dingtalk sessionWebhook rejected by SSRF policy: %w", err)
|
||||
}
|
||||
body := map[string]interface{}{
|
||||
"msgtype": "markdown",
|
||||
"markdown": map[string]string{
|
||||
|
|
|
|||
|
|
@ -7,15 +7,24 @@ import (
|
|||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
)
|
||||
|
||||
func useTestHTTPClient(t *testing.T) {
|
||||
t.Helper()
|
||||
original := httpClient
|
||||
httpClient = &http.Client{Timeout: 5 * time.Second}
|
||||
t.Cleanup(func() { httpClient = original })
|
||||
}
|
||||
|
||||
// TestDownloadFile_EndToEnd drives the full DownloadFile orchestration against a
|
||||
// fake DingTalk OpenAPI: access-token fetch → messageFiles/download (downloadCode
|
||||
// → temporary downloadUrl) → GET the temp URL for the bytes. This covers the HTTP
|
||||
// path that the pure-function unit tests deliberately skip (issue #1771).
|
||||
func TestDownloadFile_EndToEnd(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
fileBytes := []byte("%PDF-1.7 fake product spec bytes")
|
||||
|
||||
var downloadReq map[string]string
|
||||
|
|
@ -89,6 +98,7 @@ func TestDownloadFile_EndToEnd(t *testing.T) {
|
|||
// TestDownloadFile_TempURLError verifies a non-200 from the temporary download
|
||||
// URL surfaces as an error rather than silently returning empty content.
|
||||
func TestDownloadFile_TempURLError(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
|
|
@ -121,6 +131,7 @@ func TestDownloadFile_TempURLError(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDownloadFile_SSRFRejected(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,10 @@ import (
|
|||
var _ im.StreamSender = (*Adapter)(nil)
|
||||
var _ im.FileDownloader = (*Adapter)(nil)
|
||||
|
||||
var httpClient = &http.Client{Timeout: 10 * time.Second}
|
||||
var httpClient = utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 10 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
// Adapter implements im.Adapter for Feishu/Lark.
|
||||
type Adapter struct {
|
||||
|
|
|
|||
|
|
@ -8,10 +8,18 @@ import (
|
|||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
)
|
||||
|
||||
func useTestHTTPClient(t *testing.T) {
|
||||
t.Helper()
|
||||
original := httpClient
|
||||
httpClient = &http.Client{Timeout: 5 * time.Second}
|
||||
t.Cleanup(func() { httpClient = original })
|
||||
}
|
||||
|
||||
// testRegion returns a Lark-shaped region pointed at a fake Open Platform, so
|
||||
// the whole send path can be driven without touching the real cloud. Region
|
||||
// carries the host as a field precisely so this is possible.
|
||||
|
|
@ -86,6 +94,7 @@ func (f *fakeOpenPlatform) sawPath(want string) bool {
|
|||
// SendReply must fetch a token from the region's cloud and reply under the
|
||||
// original message, carrying that token as a bearer credential.
|
||||
func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
fake := &fakeOpenPlatform{}
|
||||
srv := httptest.NewServer(fake.handler())
|
||||
defer srv.Close()
|
||||
|
|
@ -130,6 +139,7 @@ func TestSendReply_EndToEnd_UsesRegionCloud(t *testing.T) {
|
|||
// A group that rejects reply-in-thread (230071) must still receive the answer
|
||||
// via the plain send-message API, addressed to the chat.
|
||||
func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
fake := &fakeOpenPlatform{replyCode: 230071}
|
||||
srv := httptest.NewServer(fake.handler())
|
||||
defer srv.Close()
|
||||
|
|
@ -160,6 +170,7 @@ func TestSendReply_EndToEnd_FallsBackToSendAPI(t *testing.T) {
|
|||
|
||||
// A non-fallback-eligible error must surface, not be silently swallowed.
|
||||
func TestSendReply_EndToEnd_HardErrorSurfaces(t *testing.T) {
|
||||
useTestHTTPClient(t)
|
||||
fake := &fakeOpenPlatform{replyCode: 99991663} // app ticket invalid — not retryable
|
||||
srv := httptest.NewServer(fake.handler())
|
||||
defer srv.Close()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
ws "github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
|
|
@ -68,7 +69,9 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, _, err := ws.DefaultDialer.DialContext(ctx, gatewayURL, nil)
|
||||
dialer := *ws.DefaultDialer
|
||||
dialer.NetDialContext = secutils.SSRFSafeDialContext
|
||||
conn, _, err := dialer.DialContext(ctx, gatewayURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// Compile-time checks.
|
||||
|
|
@ -72,8 +73,8 @@ func (a *Adapter) VerifyCallback(c *gin.Context) error {
|
|||
|
||||
// telegramUpdate represents an incoming Telegram update (subset of fields).
|
||||
type telegramUpdate struct {
|
||||
UpdateID int `json:"update_id"`
|
||||
Message *telegramMsg `json:"message"`
|
||||
UpdateID int `json:"update_id"`
|
||||
Message *telegramMsg `json:"message"`
|
||||
}
|
||||
|
||||
type telegramMsg struct {
|
||||
|
|
@ -256,7 +257,10 @@ func (a *Adapter) editMessage(ctx context.Context, chatID, messageID, text, pars
|
|||
}
|
||||
|
||||
// httpClient is a shared HTTP client with a reasonable timeout for Telegram API calls.
|
||||
var httpClient = &http.Client{Timeout: 15 * time.Second}
|
||||
var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 15 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
// callAPI calls the Telegram Bot API, discarding the result.
|
||||
func (a *Adapter) callAPI(ctx context.Context, method string, body interface{}) error {
|
||||
|
|
@ -311,7 +315,7 @@ const minEditInterval = 500 * time.Millisecond
|
|||
type streamState struct {
|
||||
mu sync.Mutex
|
||||
content strings.Builder
|
||||
msgID string // Telegram message ID of the "thinking" message
|
||||
msgID string // Telegram message ID of the "thinking" message
|
||||
chatID string
|
||||
lastEdit time.Time // last successful editMessageText timestamp
|
||||
createdAt time.Time // for orphan stream detection
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// MessageHandler is called when an IM message is received via long polling.
|
||||
|
|
@ -26,9 +27,12 @@ type LongConnClient struct {
|
|||
// NewLongConnClient creates a Telegram long-polling client.
|
||||
func NewLongConnClient(botToken string, handler MessageHandler) *LongConnClient {
|
||||
return &LongConnClient{
|
||||
botToken: botToken,
|
||||
handler: handler,
|
||||
httpClient: &http.Client{Timeout: 35 * time.Second},
|
||||
botToken: botToken,
|
||||
handler: handler,
|
||||
httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 35 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
|
|
@ -37,7 +38,10 @@ const (
|
|||
channelVersion = "weknora-1.0.0"
|
||||
)
|
||||
|
||||
var ilinkHTTPClient = &http.Client{Timeout: 30 * time.Second}
|
||||
var ilinkHTTPClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
// BuildCDNDownloadURL constructs a CDN download URL from an encrypt_query_param.
|
||||
func BuildCDNDownloadURL(encryptQueryParam string) string {
|
||||
|
|
@ -145,6 +149,9 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io
|
|||
if msg.FileKey == "" {
|
||||
return nil, "", fmt.Errorf("no file URL in message")
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(msg.FileKey); err != nil {
|
||||
return nil, "", fmt.Errorf("file URL rejected by SSRF policy: %w", err)
|
||||
}
|
||||
|
||||
fileName := msg.FileName
|
||||
if fileName == "" {
|
||||
|
|
|
|||
|
|
@ -22,13 +22,14 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
longPollTimeout = 35 * time.Second
|
||||
longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout
|
||||
reconnectBaseDelay = 1 * time.Second
|
||||
reconnectMaxDelay = 30 * time.Second
|
||||
longPollTimeout = 35 * time.Second
|
||||
longPollHTTPTimeout = 40 * time.Second // slightly longer than poll timeout
|
||||
reconnectBaseDelay = 1 * time.Second
|
||||
reconnectMaxDelay = 30 * time.Second
|
||||
maxReconnectAttempts = -1 // infinite
|
||||
)
|
||||
|
||||
|
|
@ -50,7 +51,10 @@ func NewLongPollClient(botToken, ilinkBotID string, handler func(ctx context.Con
|
|||
botToken: botToken,
|
||||
ilinkBotID: ilinkBotID,
|
||||
handler: handler,
|
||||
httpClient: &http.Client{Timeout: longPollHTTPTimeout},
|
||||
httpClient: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: longPollHTTPTimeout,
|
||||
MaxRedirects: 5,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -316,25 +320,25 @@ func pollReconnectDelay(attempt int) time.Duration {
|
|||
// ── iLink API response types (matches proto: GetUpdatesResp, WeixinMessage) ──
|
||||
|
||||
type getUpdatesResponse struct {
|
||||
Ret int `json:"ret"`
|
||||
ErrCode int `json:"errcode"`
|
||||
ErrMsg string `json:"errmsg"`
|
||||
Msgs []weixinMessage `json:"msgs"`
|
||||
GetUpdatesBuf string `json:"get_updates_buf"`
|
||||
Ret int `json:"ret"`
|
||||
ErrCode int `json:"errcode"`
|
||||
ErrMsg string `json:"errmsg"`
|
||||
Msgs []weixinMessage `json:"msgs"`
|
||||
GetUpdatesBuf string `json:"get_updates_buf"`
|
||||
}
|
||||
|
||||
type weixinMessage struct {
|
||||
Seq int `json:"seq"`
|
||||
MessageID int64 `json:"message_id"`
|
||||
FromUserID string `json:"from_user_id"`
|
||||
ToUserID string `json:"to_user_id"`
|
||||
ClientID string `json:"client_id"`
|
||||
CreateTimeMs int64 `json:"create_time_ms"`
|
||||
SessionID string `json:"session_id"`
|
||||
MessageType int `json:"message_type"` // 1=USER, 2=BOT
|
||||
MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH
|
||||
ItemList []messageItem `json:"item_list"`
|
||||
ContextToken string `json:"context_token"`
|
||||
Seq int `json:"seq"`
|
||||
MessageID int64 `json:"message_id"`
|
||||
FromUserID string `json:"from_user_id"`
|
||||
ToUserID string `json:"to_user_id"`
|
||||
ClientID string `json:"client_id"`
|
||||
CreateTimeMs int64 `json:"create_time_ms"`
|
||||
SessionID string `json:"session_id"`
|
||||
MessageType int `json:"message_type"` // 1=USER, 2=BOT
|
||||
MessageState int `json:"message_state"` // 0=NEW, 1=GENERATING, 2=FINISH
|
||||
ItemList []messageItem `json:"item_list"`
|
||||
ContextToken string `json:"context_token"`
|
||||
}
|
||||
|
||||
type messageItem struct {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
// QR code login flow for WeChat iLink Bot API.
|
||||
//
|
||||
// API endpoints (all relative to the iLink base URL):
|
||||
// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content}
|
||||
// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...}
|
||||
//
|
||||
// GET /ilink/bot/get_bot_qrcode?bot_type=3 → returns {qrcode, qrcode_img_content}
|
||||
// GET /ilink/bot/get_qrcode_status?qrcode=xxx → returns {status, bot_token, ilink_bot_id, ...}
|
||||
//
|
||||
// Flow:
|
||||
// 1. Call GetLoginQRCode to obtain a QR code URL and opaque qrcode token
|
||||
|
|
@ -19,6 +20,8 @@ import (
|
|||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// QRCodeResult holds the result of requesting a login QR code.
|
||||
|
|
@ -46,7 +49,10 @@ type QRCodeService struct {
|
|||
// NewQRCodeService creates a new QR code service.
|
||||
func NewQRCodeService() *QRCodeService {
|
||||
return &QRCodeService{
|
||||
client: &http.Client{Timeout: pollTimeout},
|
||||
client: secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: pollTimeout,
|
||||
MaxRedirects: 5,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +82,7 @@ func (s *QRCodeService) GetLoginQRCode(ctx context.Context) (*QRCodeResult, erro
|
|||
}
|
||||
|
||||
var result struct {
|
||||
QRCode string `json:"qrcode"`
|
||||
QRCode string `json:"qrcode"`
|
||||
QRCodeImgContent string `json:"qrcode_img_content"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/im"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
ws "github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
|
|
@ -365,7 +366,9 @@ func (c *LongConnClient) sendStreamFrame(incoming *im.IncomingMessage, streamID,
|
|||
}
|
||||
|
||||
func (c *LongConnClient) connectAndRun(ctx context.Context) error {
|
||||
conn, _, err := ws.DefaultDialer.DialContext(ctx, c.endpoint, nil)
|
||||
dialer := *ws.DefaultDialer
|
||||
dialer.NetDialContext = secutils.SSRFSafeDialContext
|
||||
conn, _, err := dialer.DialContext(ctx, c.endpoint, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,10 @@ import (
|
|||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
var httpClient = &http.Client{Timeout: 30 * time.Second}
|
||||
var httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
const defaultAPIBaseURL = "https://qyapi.weixin.qq.com"
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -62,16 +63,17 @@ type HTTPDocumentReader struct {
|
|||
}
|
||||
|
||||
func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) {
|
||||
baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/")
|
||||
if baseURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(baseURL); err != nil {
|
||||
return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err)
|
||||
}
|
||||
}
|
||||
clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
clientCfg.Timeout = 5 * time.Minute
|
||||
p := &HTTPDocumentReader{
|
||||
baseURL: strings.TrimSuffix(baseURL, "/"),
|
||||
client: &http.Client{
|
||||
Timeout: 5 * time.Minute,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
},
|
||||
},
|
||||
baseURL: baseURL,
|
||||
client: secutils.NewSSRFSafeHTTPClient(clientCfg),
|
||||
}
|
||||
if p.baseURL != "" {
|
||||
logger.Infof(context.Background(), "INFO: HTTP docreader base URL: %s", p.baseURL)
|
||||
|
|
@ -86,9 +88,15 @@ func (p *HTTPDocumentReader) base() string {
|
|||
}
|
||||
|
||||
func (p *HTTPDocumentReader) Reconnect(addr string) error {
|
||||
addr = strings.TrimSuffix(strings.TrimSpace(addr), "/")
|
||||
if addr != "" {
|
||||
if err := secutils.ValidateURLForSSRF(addr); err != nil {
|
||||
return fmt.Errorf("docreader address failed SSRF validation: %w", err)
|
||||
}
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.baseURL = strings.TrimSuffix(addr, "/")
|
||||
p.baseURL = addr
|
||||
logger.Infof(context.Background(), "INFO: HTTP docreader base URL set to %s", p.baseURL)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -122,6 +130,9 @@ func (p *HTTPDocumentReader) ListEngines(ctx context.Context, overrides map[stri
|
|||
if base == "" {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(base); err != nil {
|
||||
return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err)
|
||||
}
|
||||
|
||||
body := httpListEnginesRequest{ConfigOverrides: overrides}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
|
|
@ -187,6 +198,9 @@ func (p *HTTPDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (
|
|||
if base == "" {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(base); err != nil {
|
||||
return nil, fmt.Errorf("docreader address failed SSRF validation: %w", err)
|
||||
}
|
||||
|
||||
body := httpReadRequest{
|
||||
FileName: req.FileName,
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
|
|||
if c.token == "" {
|
||||
return &types.ReadResult{Error: "PaddleOCR-VL Cloud token is not configured"}, nil
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(c.baseURL); err != nil {
|
||||
return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL Cloud base URL blocked by SSRF policy: %v", err)}, nil
|
||||
}
|
||||
|
||||
content := req.FileContent
|
||||
if len(content) == 0 {
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
|
|||
if c.endpoint == "" {
|
||||
return &types.ReadResult{Error: "PaddleOCR-VL endpoint is not configured"}, nil
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(c.endpoint); err != nil {
|
||||
return &types.ReadResult{Error: fmt.Sprintf("PaddleOCR-VL endpoint blocked by SSRF policy: %v", err)}, nil
|
||||
}
|
||||
|
||||
content := req.FileContent
|
||||
if len(content) == 0 {
|
||||
|
|
@ -157,7 +160,10 @@ func (c *PaddleOCRVLReader) callLayoutParsing(
|
|||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: paddleOCRVLTimeout}
|
||||
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: paddleOCRVLTimeout,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("HTTP request: %w", err)
|
||||
|
|
@ -265,6 +271,9 @@ func PingPaddleOCRVL(endpoint string) (bool, string) {
|
|||
if endpoint == "" {
|
||||
return false, "未配置 PaddleOCR-VL 端点"
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return false, fmt.Sprintf("PaddleOCR-VL 端点未通过 SSRF 校验: %v", err)
|
||||
}
|
||||
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: 5 * time.Second,
|
||||
MaxRedirects: 5,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
|
@ -39,20 +40,15 @@ func NewWeKnoraCloudSignedDocumentReader(appID, apiKey string) (*WeKnoraCloudSig
|
|||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud apiKey is required")
|
||||
}
|
||||
clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
clientCfg.Timeout = 500 * time.Minute
|
||||
return &WeKnoraCloudSignedDocumentReader{
|
||||
appID: appID,
|
||||
apiKey: apiKey,
|
||||
initialPollInterval: 500 * time.Millisecond,
|
||||
maxPollInterval: 10 * time.Second,
|
||||
pollTimeout: 20 * time.Minute,
|
||||
client: &http.Client{
|
||||
Timeout: 500 * time.Minute,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
},
|
||||
},
|
||||
client: secutils.NewSSRFSafeHTTPClient(clientCfg),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,8 +39,9 @@ func NewBaiduProvider(params types.WebSearchProviderParameters) (interfaces.WebS
|
|||
if params.APIKey == "" {
|
||||
return nil, fmt.Errorf("API key is required for Baidu provider")
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: defaultBaiduTimeout,
|
||||
client, err := NewSearchHTTPClient(defaultBaiduTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Baidu HTTP client: %w", err)
|
||||
}
|
||||
return &BaiduProvider{
|
||||
client: client,
|
||||
|
|
|
|||
|
|
@ -35,8 +35,9 @@ func NewOllamaProvider(params types.WebSearchProviderParameters) (interfaces.Web
|
|||
if params.APIKey == "" {
|
||||
return nil, fmt.Errorf("API key is required for Ollama provider")
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: defaultOllamaTimeout,
|
||||
client, err := NewSearchHTTPClient(defaultOllamaTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Ollama HTTP client: %w", err)
|
||||
}
|
||||
return &OllamaProvider{
|
||||
client: client,
|
||||
|
|
|
|||
|
|
@ -19,18 +19,6 @@ func ValidateProxyURL(proxyURL string) error {
|
|||
return utils.ValidateURLForSSRF(proxyURL)
|
||||
}
|
||||
|
||||
func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("stopped after %d redirects", maxRedirects)
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil {
|
||||
return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewSearchHTTPClient builds an http.Client for outbound web search requests.
|
||||
// It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and
|
||||
// redirect validation consistent with utils.NewSSRFSafeHTTPClient.
|
||||
|
|
@ -60,9 +48,6 @@ func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client,
|
|||
}
|
||||
|
||||
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
return &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: t,
|
||||
CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects),
|
||||
}, nil
|
||||
cfg.Timeout = timeout
|
||||
return utils.NewSSRFSafeHTTPClientWithTransport(cfg, t), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,10 +59,13 @@ func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) {
|
||||
h := ssrfSafeRedirect(10)
|
||||
client, err := NewSearchHTTPClient(0, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &http.Request{URL: mustParse(t, "ftp://evil.com/")}
|
||||
// len(via)==0 so first redirect
|
||||
err := h(req, nil)
|
||||
err = client.CheckRedirect(req, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for ftp redirect")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/mark3labs/mcp-go/client"
|
||||
"github.com/mark3labs/mcp-go/client/transport"
|
||||
"github.com/mark3labs/mcp-go/mcp"
|
||||
|
|
@ -147,15 +148,22 @@ func asOAuthRequired(err error) *OAuthRequiredError {
|
|||
|
||||
// NewMCPClient creates a new MCP client based on the transport type
|
||||
func NewMCPClient(config *ClientConfig) (MCPClient, error) {
|
||||
if config == nil || config.Service == nil {
|
||||
return nil, fmt.Errorf("MCP client config and service are required")
|
||||
}
|
||||
if err := ValidateServiceOutboundURLs(config.Service); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create HTTP client with timeout
|
||||
timeout := 30 * time.Second
|
||||
if config.Service.AdvancedConfig != nil && config.Service.AdvancedConfig.Timeout > 0 {
|
||||
timeout = time.Duration(config.Service.AdvancedConfig.Timeout) * time.Second
|
||||
}
|
||||
|
||||
httpClient := &http.Client{
|
||||
Timeout: timeout,
|
||||
}
|
||||
clientCfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
clientCfg.Timeout = timeout
|
||||
httpClient := secutils.NewSSRFSafeHTTPClient(clientCfg)
|
||||
|
||||
// Build headers
|
||||
headers := make(map[string]string)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/mark3labs/mcp-go/client/transport"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
|
@ -52,12 +53,18 @@ func (m *OAuthManager) newHandler(
|
|||
if service.URL == nil || *service.URL == "" {
|
||||
return nil, fmt.Errorf("MCP service URL is required for OAuth")
|
||||
}
|
||||
if err := ValidateServiceOutboundURLs(service); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpCfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
httpCfg.Timeout = 30 * time.Second
|
||||
cfg := transport.OAuthConfig{
|
||||
RedirectURI: redirectURI,
|
||||
Scopes: service.AuthConfig.Scopes,
|
||||
TokenStore: newDBTokenStore(m.repo, tenantID, principal, service.ID),
|
||||
PKCEEnabled: true,
|
||||
AuthServerMetadataURL: service.AuthConfig.AuthServerMetadataURL,
|
||||
HTTPClient: secutils.NewSSRFSafeHTTPClient(httpCfg),
|
||||
}
|
||||
if existing, err := m.repo.GetClient(ctx, tenantID, service.ID); err == nil && existing != nil {
|
||||
cfg.ClientID = existing.ClientID
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
package mcp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// ValidateServiceOutboundURLs validates every URL that the MCP transport or
|
||||
// OAuth discovery flow may contact. It is intentionally called both at
|
||||
// persistence boundaries and immediately before client construction so stale
|
||||
// or imported rows cannot bypass the current SSRF policy.
|
||||
func ValidateServiceOutboundURLs(service *types.MCPService) error {
|
||||
if service == nil {
|
||||
return fmt.Errorf("MCP service is required")
|
||||
}
|
||||
if service.URL != nil {
|
||||
serviceURL := strings.TrimSpace(*service.URL)
|
||||
if serviceURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(serviceURL); err != nil {
|
||||
return fmt.Errorf("MCP service URL failed SSRF validation: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if service.AuthConfig != nil {
|
||||
metadataURL := strings.TrimSpace(service.AuthConfig.AuthServerMetadataURL)
|
||||
if metadataURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(metadataURL); err != nil {
|
||||
return fmt.Errorf("MCP OAuth metadata URL failed SSRF validation: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package mcp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestValidateServiceOutboundURLsRejectsOAuthMetadataSSRF(t *testing.T) {
|
||||
serviceURL := "https://example.com/mcp"
|
||||
err := ValidateServiceOutboundURLs(&types.MCPService{
|
||||
URL: &serviceURL,
|
||||
AuthConfig: &types.MCPAuthConfig{
|
||||
AuthServerMetadataURL: "http://169.254.169.254/latest/meta-data",
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected OAuth metadata URL to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewMCPClientRejectsStoredUnsafeURL(t *testing.T) {
|
||||
serviceURL := "http://127.0.0.1:8080/mcp"
|
||||
_, err := NewMCPClient(&ClientConfig{Service: &types.MCPService{
|
||||
URL: &serviceURL,
|
||||
TransportType: types.MCPTransportHTTPStreamable,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected stored unsafe MCP URL to be rejected at client construction")
|
||||
}
|
||||
}
|
||||
|
|
@ -72,16 +72,16 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// The SDK must use the same SSRF-safe transport as the raw HTTP paths.
|
||||
// Constructor-time URL validation alone cannot prevent DNS rebinding or a
|
||||
// later redirect to an internal address.
|
||||
sdkHTTPClient := rawHTTPClient
|
||||
// 如果指定了 CustomHeaders,则给 SDK 使用的 HTTPClient 挂一层 RoundTripper,
|
||||
// 在每个请求上自动注入这些 header(raw HTTP 路径会在发送前单独处理)。
|
||||
if len(chatConfig.CustomHeaders) > 0 {
|
||||
if httpClient, ok := config.HTTPClient.(*http.Client); ok {
|
||||
config.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, chatConfig.CustomHeaders)
|
||||
} else {
|
||||
// SDK 默认未显式设置时 HTTPClient 为 nil,此时构造一个新的注入了 header 的 client。
|
||||
config.HTTPClient = secutils.WrapHTTPClientWithHeaders(nil, chatConfig.CustomHeaders)
|
||||
}
|
||||
sdkHTTPClient = secutils.WrapHTTPClientWithHeaders(sdkHTTPClient, chatConfig.CustomHeaders)
|
||||
}
|
||||
config.HTTPClient = sdkHTTPClient
|
||||
|
||||
modelName := chatConfig.ModelName
|
||||
if chatConfig.ExtraConfig != nil {
|
||||
|
|
|
|||
|
|
@ -48,12 +48,15 @@ func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, cont
|
|||
// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout)
|
||||
// rather than http.Client.Timeout, so streaming calls are not prematurely terminated.
|
||||
// Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer.
|
||||
var rawHTTPClient = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: secutils.SSRFSafeDialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
},
|
||||
var rawHTTPTransport = &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: secutils.SSRFSafeDialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
}
|
||||
|
||||
var rawHTTPClient = secutils.NewSSRFSafeHTTPClientWithTransport(
|
||||
secutils.SSRFSafeHTTPClientConfig{Timeout: 0, MaxRedirects: 10},
|
||||
rawHTTPTransport,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
package embedding
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) {
|
||||
|
|
@ -15,10 +18,18 @@ func TestNewEmbeddingHTTPClient_ReusesTransport(t *testing.T) {
|
|||
if first == second {
|
||||
t.Fatal("expected distinct HTTP clients")
|
||||
}
|
||||
if first.Transport != second.Transport {
|
||||
t.Fatal("expected embedding HTTP clients to share a transport")
|
||||
firstGuard, ok := first.Transport.(*secutils.SSRFValidatingRoundTripper)
|
||||
if !ok {
|
||||
t.Fatalf("expected SSRF-validating transport, got %T", first.Transport)
|
||||
}
|
||||
if first.Transport != sharedEmbeddingHTTPTransport {
|
||||
secondGuard, ok := second.Transport.(*secutils.SSRFValidatingRoundTripper)
|
||||
if !ok {
|
||||
t.Fatalf("expected SSRF-validating transport, got %T", second.Transport)
|
||||
}
|
||||
if firstGuard.Base != secondGuard.Base {
|
||||
t.Fatal("expected embedding HTTP clients to share a base transport")
|
||||
}
|
||||
if firstGuard.Base != http.RoundTripper(sharedEmbeddingHTTPTransport) {
|
||||
t.Fatal("expected embedding HTTP client to use the shared transport")
|
||||
}
|
||||
if first.Timeout != firstTimeout {
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *
|
|||
return client
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{}
|
||||
client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig())
|
||||
}
|
||||
base := client.Transport
|
||||
if base == nil {
|
||||
|
|
|
|||
|
|
@ -50,6 +50,9 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWrapHTTPClientWithHeaders(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
|
||||
ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(ResetSSRFWhitelistForTest)
|
||||
gotTrace := ""
|
||||
gotAuth := ""
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
var defaultHTTPClient = &http.Client{Timeout: 60 * time.Second}
|
||||
var defaultHTTPClient = NewSSRFSafeHTTPClient(SSRFSafeHTTPClientConfig{
|
||||
Timeout: 60 * time.Second,
|
||||
MaxRedirects: 10,
|
||||
})
|
||||
|
||||
// DownloadBytes fetches the content at the given HTTP(S) URL and returns the
|
||||
// raw bytes. It reuses a package-level http.Client with a 60-second timeout.
|
||||
|
|
@ -16,6 +19,9 @@ func DownloadBytes(url string) ([]byte, error) {
|
|||
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
|
||||
return nil, fmt.Errorf("unsupported URL scheme: %s", url)
|
||||
}
|
||||
if err := ValidateURLForSSRF(url); err != nil {
|
||||
return nil, fmt.Errorf("URL rejected by SSRF policy: %w", err)
|
||||
}
|
||||
resp, err := defaultHTTPClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("HTTP GET: %w", err)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// MySQLSSRFNetwork is the mysql driver network name registered by
|
||||
// RegisterMySQLSSRFDialer. DSNs must set cfg.Net to this value so every
|
||||
// database/sql open uses SSRFSafeDialContext at the final TCP sink.
|
||||
const MySQLSSRFNetwork = "tcp-weknora-ssrf"
|
||||
|
||||
var mysqlSSRFDialerOnce sync.Once
|
||||
|
||||
// RegisterMySQLSSRFDialer installs a go-sql-driver/mysql dialer that routes
|
||||
// connections through SSRFSafeDialContext. Safe to call from multiple packages;
|
||||
// registration happens at most once per process.
|
||||
func RegisterMySQLSSRFDialer() {
|
||||
mysqlSSRFDialerOnce.Do(func() {
|
||||
mysql.RegisterDialContext(MySQLSSRFNetwork, func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
return SSRFSafeDialContext(ctx, "tcp", addr)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -254,6 +254,26 @@ var restrictedIPv4Ranges = []*net.IPNet{
|
|||
mustParseCIDR("172.20.0.0/16"),
|
||||
}
|
||||
|
||||
// restrictedPorts contains non-HTTP service ports that user-controlled URLs
|
||||
// must not reach. It is checked both during URL validation and again at dial
|
||||
// time so dynamically discovered URLs cannot bypass the input boundary.
|
||||
var restrictedPorts = map[string]bool{
|
||||
"22": true, // SSH
|
||||
"23": true, // Telnet
|
||||
"25": true, // SMTP
|
||||
"445": true, // SMB
|
||||
"3389": true, // RDP
|
||||
"5432": true, // PostgreSQL
|
||||
"3306": true, // MySQL
|
||||
"6379": true, // Redis
|
||||
"27017": true, // MongoDB
|
||||
"9200": true, // Elasticsearch
|
||||
"2379": true, // etcd
|
||||
"2380": true, // etcd
|
||||
"8500": true, // Consul
|
||||
"4001": true, // etcd (old)
|
||||
}
|
||||
|
||||
// mustParseCIDR parses a CIDR string and panics on error
|
||||
func mustParseCIDR(s string) *net.IPNet {
|
||||
_, ipNet, err := net.ParseCIDR(s)
|
||||
|
|
@ -459,27 +479,8 @@ func isSSRFSafeURL(rawURL string) (bool, string) {
|
|||
|
||||
// Check for suspicious port numbers
|
||||
port := parsed.Port()
|
||||
if port != "" {
|
||||
// Block common internal service ports
|
||||
blockedPorts := map[string]bool{
|
||||
"22": true, // SSH
|
||||
"23": true, // Telnet
|
||||
"25": true, // SMTP
|
||||
"445": true, // SMB
|
||||
"3389": true, // RDP
|
||||
"5432": true, // PostgreSQL
|
||||
"3306": true, // MySQL
|
||||
"6379": true, // Redis
|
||||
"27017": true, // MongoDB
|
||||
"9200": true, // Elasticsearch
|
||||
"2379": true, // etcd
|
||||
"2380": true, // etcd
|
||||
"8500": true, // Consul
|
||||
"4001": true, // etcd (old)
|
||||
}
|
||||
if blockedPorts[port] {
|
||||
return false, fmt.Sprintf("port %s is blocked for security reasons", port)
|
||||
}
|
||||
if restrictedPorts[port] {
|
||||
return false, fmt.Sprintf("port %s is blocked for security reasons", port)
|
||||
}
|
||||
|
||||
return true, ""
|
||||
|
|
@ -818,15 +819,36 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request)
|
|||
if redirectHost != "" && IsSSRFWhitelisted(redirectHost) {
|
||||
return nil
|
||||
}
|
||||
redirectURL := req.URL.String()
|
||||
if safe, reason := isSSRFSafeURL(redirectURL); !safe {
|
||||
return fmt.Errorf("%w: %s", ErrSSRFRedirectBlocked, reason)
|
||||
if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrSSRFRedirectBlocked, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SSRFValidatingRoundTripper enforces the URL policy for every outbound
|
||||
// request, including URLs discovered at runtime by SDKs (for example OAuth
|
||||
// metadata) that never passed through an application handler. Dial-time checks
|
||||
// remain necessary to pin DNS answers and cover transports that cannot accept
|
||||
// this wrapper directly.
|
||||
type SSRFValidatingRoundTripper struct {
|
||||
Base http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *SSRFValidatingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if req == nil || req.URL == nil {
|
||||
return nil, fmt.Errorf("outbound request blocked: request URL is required")
|
||||
}
|
||||
if t == nil || t.Base == nil {
|
||||
return nil, fmt.Errorf("outbound request blocked: base transport is required")
|
||||
}
|
||||
if err := validateURLForSSRFForOutbound(req.URL.String()); err != nil {
|
||||
return nil, fmt.Errorf("outbound request blocked by SSRF policy: %w", err)
|
||||
}
|
||||
return t.Base.RoundTrip(req)
|
||||
}
|
||||
|
||||
// NewSSRFSafeHTTPClientWithTransport wraps a caller-supplied transport in an
|
||||
// *http.Client carrying the given timeout and the SSRF-aware redirect policy.
|
||||
// Pass a transport from NewSSRFSafeTransport (optionally shared across clients)
|
||||
|
|
@ -834,9 +856,12 @@ func newSSRFCheckRedirect(maxRedirects int) func(*http.Request, []*http.Request)
|
|||
func NewSSRFSafeHTTPClientWithTransport(
|
||||
config SSRFSafeHTTPClientConfig, transport http.RoundTripper,
|
||||
) *http.Client {
|
||||
if transport == nil {
|
||||
transport = NewSSRFSafeTransport(config)
|
||||
}
|
||||
return &http.Client{
|
||||
Timeout: config.Timeout,
|
||||
Transport: transport,
|
||||
Transport: &SSRFValidatingRoundTripper{Base: transport},
|
||||
CheckRedirect: newSSRFCheckRedirect(config.MaxRedirects),
|
||||
}
|
||||
}
|
||||
|
|
@ -849,12 +874,18 @@ func NewSSRFSafeHTTPClient(config SSRFSafeHTTPClientConfig) *http.Client {
|
|||
return NewSSRFSafeHTTPClientWithTransport(config, NewSSRFSafeTransport(config))
|
||||
}
|
||||
|
||||
// SSRFSafeGRPCDialer is compatible with grpc.WithContextDialer and pins DNS
|
||||
// answers the same way as SSRFSafeDialContext.
|
||||
func SSRFSafeGRPCDialer(ctx context.Context, addr string) (net.Conn, error) {
|
||||
return SSRFSafeDialContext(ctx, "tcp", addr)
|
||||
}
|
||||
|
||||
// SSRFSafeDialContext is a custom dial function that validates the resolved IP addresses
|
||||
// before establishing a connection. This provides an additional layer of SSRF protection
|
||||
// against DNS rebinding attacks during the connection phase.
|
||||
func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
// Parse host and port
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid address %s: %w", addr, err)
|
||||
}
|
||||
|
|
@ -870,6 +901,9 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e
|
|||
}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
}
|
||||
if restrictedPorts[port] {
|
||||
return nil, fmt.Errorf("connection blocked: port %s is restricted", port)
|
||||
}
|
||||
|
||||
// Check if the host is a restricted hostname
|
||||
hostLower := strings.ToLower(host)
|
||||
|
|
@ -884,11 +918,17 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e
|
|||
}
|
||||
}
|
||||
|
||||
// Resolve the hostname to IP addresses
|
||||
// Resolve the hostname once, validate every answer, and then dial one of
|
||||
// those exact IPs. Dialing the original hostname here would make the
|
||||
// standard dialer resolve it a second time, leaving a DNS-rebinding window
|
||||
// between validation and connection establishment.
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("DNS resolution failed for %s: %w", host, err)
|
||||
}
|
||||
if len(ips) == 0 {
|
||||
return nil, fmt.Errorf("DNS resolution returned no addresses for %s", host)
|
||||
}
|
||||
|
||||
// Validate all resolved IPs
|
||||
for _, ipAddr := range ips {
|
||||
|
|
@ -897,13 +937,22 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e
|
|||
}
|
||||
}
|
||||
|
||||
// If we get here, all IPs are safe. Connect using the standard dialer.
|
||||
// We dial the original address so that proper connection routing happens.
|
||||
// If we get here, all IPs are safe. Pin the connection to the validated DNS
|
||||
// answers; TLS still uses the request hostname for SNI/certificate checks.
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
var lastErr error
|
||||
for _, ipAddr := range ips {
|
||||
pinnedAddr := net.JoinHostPort(ipAddr.IP.String(), port)
|
||||
conn, dialErr := dialer.DialContext(ctx, network, pinnedAddr)
|
||||
if dialErr == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = dialErr
|
||||
}
|
||||
return nil, fmt.Errorf("failed to connect to validated addresses for %s: %w", host, lastErr)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -987,6 +1036,7 @@ func loadSSRFWhitelist() *ssrfWhitelistConfig {
|
|||
// applySSRFWhitelist for the canonical merge logic.
|
||||
func SetSSRFWhitelistFromRaw(raw string) {
|
||||
ssrfWhitelistAtomic.Store(parseSSRFWhitelistRaw(raw))
|
||||
invalidateSSRFOutboundValidationCache()
|
||||
}
|
||||
|
||||
// parseSSRFWhitelistRaw parses a comma-separated whitelist string into
|
||||
|
|
@ -1165,6 +1215,7 @@ func ResetSSRFWhitelistForTest() {
|
|||
ssrfWhitelistOnce = sync.Once{}
|
||||
ssrfWhitelist = nil
|
||||
ssrfWhitelistAtomic.Store(nil)
|
||||
invalidateSSRFOutboundValidationCache()
|
||||
}
|
||||
|
||||
// FormatSSRFError takes the error returned by ValidateURLForSSRF and wraps
|
||||
|
|
@ -1242,6 +1293,13 @@ func ValidateURLForSSRF(rawURL string) error {
|
|||
return fmt.Errorf("URL has no hostname")
|
||||
}
|
||||
|
||||
// A whitelist relaxes host/IP restrictions only. It must never turn other
|
||||
// schemes (file://, gopher://, etc.) into valid outbound request targets.
|
||||
scheme := strings.ToLower(parsed.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return fmt.Errorf("invalid scheme: %s (only http/https allowed)", scheme)
|
||||
}
|
||||
|
||||
// If the host is whitelisted, skip the heavy checks.
|
||||
if IsSSRFWhitelisted(hostname) {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -191,6 +191,12 @@ func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) {
|
|||
rawURL: "https://8.8.8.8/dns-query",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "whitelist does not allow non-http scheme",
|
||||
whitelist: "example.com",
|
||||
rawURL: "gopher://example.com/_stats",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,46 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type recordingRoundTripper struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
func (r *recordingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
r.called = true
|
||||
return nil, fmt.Errorf("unexpected network call")
|
||||
}
|
||||
|
||||
func TestSSRFSafeClientValidatesInitialRequestAtFinalSink(t *testing.T) {
|
||||
base := &recordingRoundTripper{}
|
||||
client := NewSSRFSafeHTTPClientWithTransport(DefaultSSRFSafeHTTPClientConfig(), base)
|
||||
req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.Do(req)
|
||||
if err == nil || !strings.Contains(err.Error(), "SSRF") {
|
||||
t.Fatalf("expected final-sink SSRF rejection, got %v", err)
|
||||
}
|
||||
if base.called {
|
||||
t.Fatal("unsafe request reached the base transport")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSRFSafeDialContextRejectsRestrictedPortAtFinalSink(t *testing.T) {
|
||||
_, err := SSRFSafeDialContext(context.Background(), "tcp", "example.com:6379")
|
||||
if err == nil || !strings.Contains(err.Error(), "port 6379") {
|
||||
t.Fatalf("expected restricted-port error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewSSRFSafeTransport_SharedAcrossClients verifies that a single transport
|
||||
// can back multiple clients (global connection pooling) while each client keeps
|
||||
// its own timeout and a redirect policy.
|
||||
|
|
@ -22,11 +57,16 @@ func TestNewSSRFSafeTransport_SharedAcrossClients(t *testing.T) {
|
|||
if first == second {
|
||||
t.Fatal("expected distinct HTTP clients")
|
||||
}
|
||||
if first.Transport != second.Transport {
|
||||
t.Fatal("expected clients to share the same transport")
|
||||
firstGuard, ok := first.Transport.(*SSRFValidatingRoundTripper)
|
||||
if !ok {
|
||||
t.Fatalf("expected SSRF-validating wrapper, got %T", first.Transport)
|
||||
}
|
||||
if first.Transport != http.RoundTripper(shared) {
|
||||
t.Fatal("expected client to use the supplied shared transport")
|
||||
secondGuard, ok := second.Transport.(*SSRFValidatingRoundTripper)
|
||||
if !ok {
|
||||
t.Fatalf("expected SSRF-validating wrapper, got %T", second.Transport)
|
||||
}
|
||||
if firstGuard.Base != secondGuard.Base || firstGuard.Base != http.RoundTripper(shared) {
|
||||
t.Fatal("expected clients to share the supplied base transport")
|
||||
}
|
||||
if first.Timeout != 15*time.Second {
|
||||
t.Fatalf("unexpected first timeout: got %v, want %v", first.Timeout, 15*time.Second)
|
||||
|
|
@ -46,8 +86,12 @@ func TestNewSSRFSafeHTTPClient_HasDedicatedTransport(t *testing.T) {
|
|||
if client.Transport == nil {
|
||||
t.Fatal("expected a transport to be set")
|
||||
}
|
||||
if _, ok := client.Transport.(*http.Transport); !ok {
|
||||
t.Fatalf("expected *http.Transport, got %T", client.Transport)
|
||||
guard, ok := client.Transport.(*SSRFValidatingRoundTripper)
|
||||
if !ok {
|
||||
t.Fatalf("expected SSRF-validating wrapper, got %T", client.Transport)
|
||||
}
|
||||
if _, ok := guard.Base.(*http.Transport); !ok {
|
||||
t.Fatalf("expected *http.Transport base, got %T", guard.Base)
|
||||
}
|
||||
if client.CheckRedirect == nil {
|
||||
t.Fatal("expected SSRF redirect policy to be set")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// outboundSSRFValidationTTL bounds how long a successful RoundTrip/redirect
|
||||
// validation for the same origin may be reused. DNS rebinding at the TCP sink
|
||||
// is still blocked by SSRFSafeDialContext on every new connection.
|
||||
const outboundSSRFValidationTTL = 60 * time.Second
|
||||
|
||||
var (
|
||||
ssrfOutboundCacheGen atomic.Uint64
|
||||
ssrfOutboundCache sync.Map // string -> *ssrfOutboundCacheEntry
|
||||
ssrfOutboundValidateGroup singleflight.Group
|
||||
ssrfOutboundValidateMisses atomic.Uint64 // test-only counter
|
||||
)
|
||||
|
||||
type ssrfOutboundCacheEntry struct {
|
||||
err error
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
// validateURLForSSRFForOutbound validates URLs on the hot outbound path
|
||||
// (RoundTripper, redirect checks). Results are cached per origin for a short
|
||||
// TTL so high-frequency clients do not repeat DNS lookups on every request.
|
||||
// Handler/input boundaries should keep calling ValidateURLForSSRF directly.
|
||||
func validateURLForSSRFForOutbound(rawURL string) error {
|
||||
if rawURL == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
cacheKey, ok := outboundSSRFCacheKey(rawURL)
|
||||
if !ok {
|
||||
return ValidateURLForSSRF(rawURL)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if cached, ok := ssrfOutboundCache.Load(cacheKey); ok {
|
||||
entry := cached.(*ssrfOutboundCacheEntry)
|
||||
if now.Before(entry.expiresAt) {
|
||||
return entry.err
|
||||
}
|
||||
ssrfOutboundCache.Delete(cacheKey)
|
||||
}
|
||||
|
||||
result, err, _ := ssrfOutboundValidateGroup.Do(cacheKey, func() (any, error) {
|
||||
if cached, ok := ssrfOutboundCache.Load(cacheKey); ok {
|
||||
entry := cached.(*ssrfOutboundCacheEntry)
|
||||
if time.Now().Before(entry.expiresAt) {
|
||||
return entry.err, entry.err
|
||||
}
|
||||
}
|
||||
|
||||
ssrfOutboundValidateMisses.Add(1)
|
||||
validationErr := ValidateURLForSSRF(rawURL)
|
||||
ssrfOutboundCache.Store(cacheKey, &ssrfOutboundCacheEntry{
|
||||
err: validationErr,
|
||||
expiresAt: time.Now().Add(outboundSSRFValidationTTL),
|
||||
})
|
||||
return validationErr, validationErr
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if validationErr, ok := result.(error); ok {
|
||||
return validationErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outboundSSRFCacheKey(rawURL string) (string, bool) {
|
||||
normalized := rawURL
|
||||
if !strings.Contains(normalized, "://") {
|
||||
normalized = "https://" + normalized
|
||||
}
|
||||
parsed, err := url.Parse(normalized)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
scheme := strings.ToLower(parsed.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if host == "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
port := parsed.Port()
|
||||
if port == "" {
|
||||
if scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
var origin string
|
||||
if strings.Contains(host, ":") {
|
||||
origin = fmt.Sprintf("%s://[%s]:%s", scheme, host, port)
|
||||
} else {
|
||||
origin = fmt.Sprintf("%s://%s:%s", scheme, host, port)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%d|%s", ssrfOutboundCacheGen.Load(), origin), true
|
||||
}
|
||||
|
||||
func invalidateSSRFOutboundValidationCache() {
|
||||
ssrfOutboundCacheGen.Add(1)
|
||||
ssrfOutboundCache = sync.Map{}
|
||||
ssrfOutboundValidateGroup = singleflight.Group{}
|
||||
}
|
||||
|
||||
// ResetSSRFOutboundValidationCacheForTest clears the outbound validation cache.
|
||||
// NOT for production use.
|
||||
func ResetSSRFOutboundValidationCacheForTest() {
|
||||
invalidateSSRFOutboundValidationCache()
|
||||
ssrfOutboundValidateMisses.Store(0)
|
||||
}
|
||||
|
||||
// SSRFOutboundValidationMissesForTest returns how many uncached outbound
|
||||
// validations have run since the last ResetSSRFOutboundValidationCacheForTest.
|
||||
func SSRFOutboundValidationMissesForTest() uint64 {
|
||||
return ssrfOutboundValidateMisses.Load()
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOutboundSSRFValidationCacheReusesSameOrigin(t *testing.T) {
|
||||
ResetSSRFWhitelistForTest()
|
||||
ResetSSRFOutboundValidationCacheForTest()
|
||||
SetSSRFWhitelistFromRaw("cache-test.example.com")
|
||||
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/a"); err != nil {
|
||||
t.Fatalf("first validation failed: %v", err)
|
||||
}
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/path/b"); err != nil {
|
||||
t.Fatalf("second validation failed: %v", err)
|
||||
}
|
||||
if got := SSRFOutboundValidationMissesForTest(); got != 1 {
|
||||
t.Fatalf("expected 1 cache miss for same origin, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutboundSSRFValidationCacheInvalidatesOnWhitelistChange(t *testing.T) {
|
||||
ResetSSRFWhitelistForTest()
|
||||
ResetSSRFOutboundValidationCacheForTest()
|
||||
SetSSRFWhitelistFromRaw("cache-test.example.com")
|
||||
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil {
|
||||
t.Fatalf("initial validation failed: %v", err)
|
||||
}
|
||||
if SSRFOutboundValidationMissesForTest() != 1 {
|
||||
t.Fatalf("expected one miss before whitelist change, got %d", SSRFOutboundValidationMissesForTest())
|
||||
}
|
||||
|
||||
SetSSRFWhitelistFromRaw("cache-test.example.com,other.example.com")
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil {
|
||||
t.Fatalf("validation after whitelist change failed: %v", err)
|
||||
}
|
||||
if got := SSRFOutboundValidationMissesForTest(); got != 2 {
|
||||
t.Fatalf("expected cache miss after whitelist bump, got %d misses", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutboundSSRFValidationCacheExpires(t *testing.T) {
|
||||
ResetSSRFWhitelistForTest()
|
||||
ResetSSRFOutboundValidationCacheForTest()
|
||||
SetSSRFWhitelistFromRaw("cache-test.example.com")
|
||||
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/a"); err != nil {
|
||||
t.Fatalf("initial validation failed: %v", err)
|
||||
}
|
||||
|
||||
cacheKey, ok := outboundSSRFCacheKey("https://cache-test.example.com/a")
|
||||
if !ok {
|
||||
t.Fatal("expected cache key")
|
||||
}
|
||||
raw, ok := ssrfOutboundCache.Load(cacheKey)
|
||||
if !ok {
|
||||
t.Fatal("expected cached entry")
|
||||
}
|
||||
entry := raw.(*ssrfOutboundCacheEntry)
|
||||
entry.expiresAt = time.Now().Add(-time.Second)
|
||||
ssrfOutboundCache.Store(cacheKey, entry)
|
||||
|
||||
if err := validateURLForSSRFForOutbound("https://cache-test.example.com/b"); err != nil {
|
||||
t.Fatalf("validation after expiry failed: %v", err)
|
||||
}
|
||||
if got := SSRFOutboundValidationMissesForTest(); got != 2 {
|
||||
t.Fatalf("expected revalidation after TTL expiry, got %d misses", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSRFValidatingRoundTripperUsesOutboundCache(t *testing.T) {
|
||||
ResetSSRFWhitelistForTest()
|
||||
ResetSSRFOutboundValidationCacheForTest()
|
||||
SetSSRFWhitelistFromRaw("cache-test.example.com")
|
||||
|
||||
base := &recordingRoundTripper{}
|
||||
rt := &SSRFValidatingRoundTripper{Base: base}
|
||||
|
||||
req1, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/a", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req2, err := http.NewRequest(http.MethodGet, "https://cache-test.example.com/b", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := rt.RoundTrip(req1); err == nil {
|
||||
t.Fatal("expected base transport error, validation should pass")
|
||||
}
|
||||
if _, err := rt.RoundTrip(req2); err == nil {
|
||||
t.Fatal("expected base transport error, validation should pass")
|
||||
}
|
||||
if got := SSRFOutboundValidationMissesForTest(); got != 1 {
|
||||
t.Fatalf("round tripper expected one outbound validation miss, got %d", got)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue