HTTP: prefer pip-system-certs/certifi_win32 bundle; use init-time verify in retries; add tests
This commit is contained in:
167
API/HTTP.py
167
API/HTTP.py
@@ -14,7 +14,8 @@ import sys
|
||||
import time
|
||||
import traceback
|
||||
import re
|
||||
from typing import Optional, Dict, Any, Callable, BinaryIO, List, Iterable, Set
|
||||
import os
|
||||
from typing import Optional, Dict, Any, Callable, BinaryIO, List, Iterable, Set, Union
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlparse, parse_qs
|
||||
import logging
|
||||
@@ -30,6 +31,116 @@ except Exception: # pragma: no cover - optional dependency
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_verify_value(verify_ssl: bool) -> Union[bool, str]:
|
||||
"""Return the httpx verify argument, preferring system-aware bundles.
|
||||
|
||||
Order of precedence:
|
||||
1. If verify_ssl is not True (False or path), return it.
|
||||
2. Respect existing SSL_CERT_FILE env var if present.
|
||||
3. Prefer `pip_system_certs` if present and it exposes a bundle path.
|
||||
4. Prefer `certifi_win32`/similar helpers by invoking them and reading certifi.where().
|
||||
5. Fall back to `certifi.where()` if available.
|
||||
6. Otherwise, return True to let httpx use system defaults.
|
||||
"""
|
||||
if verify_ssl is not True:
|
||||
return verify_ssl
|
||||
|
||||
env_cert = os.environ.get("SSL_CERT_FILE")
|
||||
if env_cert:
|
||||
return env_cert
|
||||
|
||||
def _try_module_bundle(mod_name: str) -> Optional[str]:
|
||||
try:
|
||||
mod = __import__(mod_name)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Common APIs that return a bundle path
|
||||
for attr in ("where", "get_ca_bundle", "bundle_path", "get_bundle_path", "get_bundle"):
|
||||
fn = getattr(mod, attr, None)
|
||||
if callable(fn):
|
||||
try:
|
||||
res = fn()
|
||||
if res:
|
||||
return res
|
||||
except Exception:
|
||||
continue
|
||||
elif isinstance(fn, str) and fn:
|
||||
return fn
|
||||
|
||||
# Some helpers (e.g., certifi_win32) expose an action to merge system certs
|
||||
for call_attr in ("add_windows_store_certs", "add_system_certs", "merge_system_certs"):
|
||||
fn = getattr(mod, call_attr, None)
|
||||
if callable(fn):
|
||||
try:
|
||||
fn()
|
||||
try:
|
||||
import certifi as _certifi
|
||||
|
||||
res = _certifi.where()
|
||||
if res:
|
||||
return res
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
# Prefer pip_system_certs if available
|
||||
for mod_name in ("pip_system_certs",):
|
||||
path = _try_module_bundle(mod_name)
|
||||
if path:
|
||||
try:
|
||||
os.environ["SSL_CERT_FILE"] = path
|
||||
except Exception:
|
||||
pass
|
||||
logger.info(f"SSL_CERT_FILE not set; using bundle from {mod_name}: {path}")
|
||||
return path
|
||||
|
||||
# Special-case helpers that merge system certs (eg. certifi_win32)
|
||||
try:
|
||||
import certifi_win32 as _cw # type: ignore
|
||||
if hasattr(_cw, "add_windows_store_certs") and callable(_cw.add_windows_store_certs):
|
||||
try:
|
||||
_cw.add_windows_store_certs()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import certifi # type: ignore
|
||||
|
||||
path = certifi.where()
|
||||
if path:
|
||||
try:
|
||||
os.environ["SSL_CERT_FILE"] = path
|
||||
except Exception:
|
||||
pass
|
||||
logger.info(
|
||||
f"SSL_CERT_FILE not set; using certifi bundle after certifi_win32: {path}"
|
||||
)
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to certifi
|
||||
try:
|
||||
import certifi # type: ignore
|
||||
|
||||
path = certifi.where()
|
||||
if path:
|
||||
try:
|
||||
os.environ["SSL_CERT_FILE"] = path
|
||||
except Exception:
|
||||
pass
|
||||
logger.info(f"SSL_CERT_FILE not set; using certifi bundle: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
# Default configuration
|
||||
DEFAULT_TIMEOUT = 30.0
|
||||
DEFAULT_RETRIES = 3
|
||||
@@ -65,11 +176,13 @@ class HTTPClient:
|
||||
self.base_headers = headers or {}
|
||||
self._client: Optional[httpx.Client] = None
|
||||
|
||||
self._httpx_verify = _resolve_verify_value(verify_ssl)
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
self._client = httpx.Client(
|
||||
timeout=self.timeout,
|
||||
verify=self.verify_ssl,
|
||||
verify=self._httpx_verify,
|
||||
headers=self._get_headers(),
|
||||
)
|
||||
return self
|
||||
@@ -351,6 +464,53 @@ class HTTPClient:
|
||||
logger.warning(
|
||||
f"Connection error on attempt {attempt + 1}/{self.retries}: {url} - {e}"
|
||||
)
|
||||
|
||||
# Detect certificate verification failures in the underlying error
|
||||
msg = str(e or "").lower()
|
||||
if ("certificate verify failed" in msg or "unable to get local issuer certificate" in msg):
|
||||
logger.info("Certificate verification failed; attempting to retry with a system-aware CA bundle")
|
||||
try:
|
||||
import httpx as _httpx
|
||||
# Use the client's precomputed verify argument (set at init)
|
||||
verify_override = self._httpx_verify
|
||||
with _httpx.Client(timeout=self.timeout, verify=verify_override, headers=self._get_headers()) as temp_client:
|
||||
try:
|
||||
response = temp_client.request(method, url, **kwargs)
|
||||
if raise_for_status:
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except Exception as e2:
|
||||
last_exception = e2
|
||||
except Exception:
|
||||
# certifi/pip-system-certs/httpx not available; fall back to existing retry behavior
|
||||
pass
|
||||
|
||||
if attempt < self.retries - 1:
|
||||
continue
|
||||
except Exception as e:
|
||||
# Catch-all to handle non-httpx exceptions that may represent
|
||||
# certificate verification failures from underlying transports.
|
||||
last_exception = e
|
||||
logger.warning(f"Request exception on attempt {attempt + 1}/{self.retries}: {url} - {e}")
|
||||
msg = str(e or "").lower()
|
||||
if ("certificate verify failed" in msg or "unable to get local issuer certificate" in msg):
|
||||
logger.info("Certificate verification failed; attempting to retry with a system-aware CA bundle")
|
||||
try:
|
||||
import httpx as _httpx
|
||||
# Use the client's precomputed verify argument (set at init)
|
||||
verify_override = self._httpx_verify
|
||||
with _httpx.Client(timeout=self.timeout, verify=verify_override, headers=self._get_headers()) as temp_client:
|
||||
try:
|
||||
response = temp_client.request(method, url, **kwargs)
|
||||
if raise_for_status:
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except Exception as e2:
|
||||
last_exception = e2
|
||||
except Exception:
|
||||
# certifi/pip-system-certs/httpx not available; fall back to existing retry behavior
|
||||
pass
|
||||
|
||||
if attempt < self.retries - 1:
|
||||
continue
|
||||
|
||||
@@ -761,12 +921,13 @@ class AsyncHTTPClient:
|
||||
self.verify_ssl = verify_ssl
|
||||
self.base_headers = headers or {}
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
self._httpx_verify = _resolve_verify_value(verify_ssl)
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Async context manager entry."""
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=self.timeout,
|
||||
verify=self.verify_ssl,
|
||||
verify=self._httpx_verify,
|
||||
headers=self._get_headers(),
|
||||
)
|
||||
return self
|
||||
|
||||
Reference in New Issue
Block a user