from __future__ import annotations import base64 import gzip import json import ssl import zlib from dataclasses import dataclass from typing import Any, Dict, Mapping, Optional, Tuple, Union from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urljoin from urllib.request import Request, urlopen __all__ = ["HttpClient", "HttpResponse", "HttpError", "DEFAULT_TIMEOUT"] from redis.backoff import ExponentialWithJitterBackoff from redis.retry import Retry from redis.utils import dummy_fail DEFAULT_USER_AGENT = "HttpClient/1.0 (+https://example.invalid)" DEFAULT_TIMEOUT = 30.0 RETRY_STATUS_CODES = {429, 500, 502, 503, 504} @dataclass class HttpResponse: status: int headers: Dict[str, str] url: str content: bytes def text(self, encoding: Optional[str] = None) -> str: enc = encoding or self._get_encoding() return self.content.decode(enc, errors="replace") def json(self) -> Any: return json.loads(self.text(encoding=self._get_encoding())) def _get_encoding(self) -> str: # Try to infer encoding from headers; default to utf-8 ctype = self.headers.get("content-type", "") # Example: application/json; charset=utf-8 for part in ctype.split(";"): p = part.strip() if p.lower().startswith("charset="): return p.split("=", 1)[1].strip() or "utf-8" return "utf-8" class HttpError(Exception): def __init__(self, status: int, url: str, message: Optional[str] = None): self.status = status self.url = url self.message = message or f"HTTP {status} for {url}" super().__init__(self.message) class HttpClient: """ A lightweight HTTP client for REST API calls. """ def __init__( self, base_url: str = "", headers: Optional[Mapping[str, str]] = None, timeout: float = DEFAULT_TIMEOUT, retry: Retry = Retry( backoff=ExponentialWithJitterBackoff(base=1, cap=10), retries=3 ), verify_tls: bool = True, # TLS verification (server) options ca_file: Optional[str] = None, ca_path: Optional[str] = None, ca_data: Optional[Union[str, bytes]] = None, # Mutual TLS (client cert) options client_cert_file: Optional[str] = None, client_key_file: Optional[str] = None, client_key_password: Optional[str] = None, auth_basic: Optional[Tuple[str, str]] = None, # (username, password) user_agent: str = DEFAULT_USER_AGENT, ) -> None: """ Initialize a new HTTP client instance. Args: base_url: Base URL for all requests. Will be prefixed to all paths. headers: Default headers to include in all requests. timeout: Default timeout in seconds for requests. retry: Retry configuration for failed requests. verify_tls: Whether to verify TLS certificates. ca_file: Path to CA certificate file for TLS verification. ca_path: Path to a directory containing CA certificates. ca_data: CA certificate data as string or bytes. client_cert_file: Path to client certificate for mutual TLS. client_key_file: Path to a client private key for mutual TLS. client_key_password: Password for an encrypted client private key. auth_basic: Tuple of (username, password) for HTTP basic auth. user_agent: User-Agent header value for requests. The client supports both regular HTTPS with server verification and mutual TLS authentication. For server verification, provide CA certificate information via ca_file, ca_path or ca_data. For mutual TLS, additionally provide a client certificate and key via client_cert_file and client_key_file. """ self.base_url = ( base_url.rstrip() + "/" if base_url and not base_url.endswith("/") else base_url ) self._default_headers = {k.lower(): v for k, v in (headers or {}).items()} self.timeout = timeout self.retry = retry self.retry.update_supported_errors((HTTPError, URLError, ssl.SSLError)) self.verify_tls = verify_tls # TLS settings self.ca_file = ca_file self.ca_path = ca_path self.ca_data = ca_data self.client_cert_file = client_cert_file self.client_key_file = client_key_file self.client_key_password = client_key_password self.auth_basic = auth_basic self.user_agent = user_agent # Public JSON-centric helpers def get( self, path: str, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: return self._json_call( "GET", path, params=params, headers=headers, timeout=timeout, body=None, expect_json=expect_json, ) def delete( self, path: str, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: return self._json_call( "DELETE", path, params=params, headers=headers, timeout=timeout, body=None, expect_json=expect_json, ) def post( self, path: str, json_body: Optional[Any] = None, data: Optional[Union[bytes, str]] = None, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: return self._json_call( "POST", path, params=params, headers=headers, timeout=timeout, body=self._prepare_body(json_body=json_body, data=data), expect_json=expect_json, ) def put( self, path: str, json_body: Optional[Any] = None, data: Optional[Union[bytes, str]] = None, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: return self._json_call( "PUT", path, params=params, headers=headers, timeout=timeout, body=self._prepare_body(json_body=json_body, data=data), expect_json=expect_json, ) def patch( self, path: str, json_body: Optional[Any] = None, data: Optional[Union[bytes, str]] = None, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: return self._json_call( "PATCH", path, params=params, headers=headers, timeout=timeout, body=self._prepare_body(json_body=json_body, data=data), expect_json=expect_json, ) # Low-level request def request( self, method: str, path: str, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, body: Optional[Union[bytes, str]] = None, timeout: Optional[float] = None, ) -> HttpResponse: url = self._build_url(path, params) all_headers = self._prepare_headers(headers, body) data = body.encode("utf-8") if isinstance(body, str) else body req = Request(url=url, method=method.upper(), data=data, headers=all_headers) context: Optional[ssl.SSLContext] = None if url.lower().startswith("https"): if self.verify_tls: # Use provided CA material if any; fall back to system defaults context = ssl.create_default_context( cafile=self.ca_file, capath=self.ca_path, cadata=self.ca_data, ) # Load client certificate for mTLS if configured if self.client_cert_file: context.load_cert_chain( certfile=self.client_cert_file, keyfile=self.client_key_file, password=self.client_key_password, ) else: # Verification disabled context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE try: return self.retry.call_with_retry( lambda: self._make_request(req, context=context, timeout=timeout), lambda _: dummy_fail(), lambda error: self._is_retryable_http_error(error), ) except HTTPError as e: # Read error body, build response, and decide on retry err_body = b"" try: err_body = e.read() except Exception: pass headers_map = {k.lower(): v for k, v in (e.headers or {}).items()} err_body = self._maybe_decompress(err_body, headers_map) status = getattr(e, "code", 0) or 0 response = HttpResponse( status=status, headers=headers_map, url=url, content=err_body, ) return response def _make_request( self, request: Request, context: Optional[ssl.SSLContext] = None, timeout: Optional[float] = None, ): with urlopen(request, timeout=timeout or self.timeout, context=context) as resp: raw = resp.read() headers_map = {k.lower(): v for k, v in resp.headers.items()} raw = self._maybe_decompress(raw, headers_map) return HttpResponse( status=resp.status, headers=headers_map, url=resp.geturl(), content=raw, ) def _is_retryable_http_error(self, error: Exception) -> bool: if isinstance(error, HTTPError): return self._should_retry_status(error.code) return False # Internal utilities def _json_call( self, method: str, path: str, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, headers: Optional[Mapping[str, str]] = None, timeout: Optional[float] = None, body: Optional[Union[bytes, str]] = None, expect_json: bool = True, ) -> Union[HttpResponse, Any]: resp = self.request( method=method, path=path, params=params, headers=headers, body=body, timeout=timeout, ) if not (200 <= resp.status < 400): raise HttpError(resp.status, resp.url, resp.text()) if expect_json: return resp.json() return resp def _prepare_body( self, json_body: Optional[Any] = None, data: Optional[Union[bytes, str]] = None ) -> Optional[Union[bytes, str]]: if json_body is not None and data is not None: raise ValueError("Provide either json_body or data, not both.") if json_body is not None: return json.dumps(json_body, ensure_ascii=False, separators=(",", ":")) return data def _build_url( self, path: str, params: Optional[ Mapping[str, Union[None, str, int, float, bool, list, tuple]] ] = None, ) -> str: url = urljoin(self.base_url or "", path) if params: # urlencode with doseq=True supports list/tuple values query = urlencode( {k: v for k, v in params.items() if v is not None}, doseq=True ) separator = "&" if ("?" in url) else "?" url = f"{url}{separator}{query}" if query else url return url def _prepare_headers( self, headers: Optional[Mapping[str, str]], body: Optional[Union[bytes, str]] ) -> Dict[str, str]: # Start with defaults prepared: Dict[str, str] = {} prepared.update(self._default_headers) # Standard defaults for JSON REST usage prepared.setdefault("accept", "application/json") prepared.setdefault("user-agent", self.user_agent) # We will send gzip accept-encoding; handle decompression manually prepared.setdefault("accept-encoding", "gzip, deflate") # If we have a string body and content-type not specified, assume JSON if body is not None and isinstance(body, str): prepared.setdefault("content-type", "application/json; charset=utf-8") # Basic authentication if provided and not overridden if self.auth_basic and "authorization" not in prepared: user, pwd = self.auth_basic token = base64.b64encode(f"{user}:{pwd}".encode("utf-8")).decode("ascii") prepared["authorization"] = f"Basic {token}" # Merge per-call headers (case-insensitive) if headers: for k, v in headers.items(): prepared[k.lower()] = v # urllib expects header keys in canonical capitalization sometimes; but it’s tolerant. # We'll return as provided; urllib will handle it. return prepared def _should_retry_status(self, status: int) -> bool: return status in RETRY_STATUS_CODES def _maybe_decompress(self, content: bytes, headers: Mapping[str, str]) -> bytes: if not content: return content encoding = (headers.get("content-encoding") or "").lower() try: if "gzip" in encoding: return gzip.decompress(content) if "deflate" in encoding: # Try raw deflate, then zlib-wrapped try: return zlib.decompress(content, -zlib.MAX_WBITS) except zlib.error: return zlib.decompress(content) except Exception: # If decompression fails, return original bytes return content return content