df

API/HTTP.py

@@ -24,7 +24,7 @@ DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.
 
 class HTTPClient:
     """Unified HTTP client with sync support."""
-    
+
     def __init__(
         self,
         timeout: float = DEFAULT_TIMEOUT,
@@ -35,7 +35,7 @@ class HTTPClient:
     ):
         """
         Initialize HTTP client.
-        
+
         Args:
             timeout: Request timeout in seconds
             retries: Number of retries on transient failures
@@ -49,7 +49,7 @@ class HTTPClient:
         self.verify_ssl = verify_ssl
         self.base_headers = headers or {}
         self._client: Optional[httpx.Client] = None
-    
+
     def __enter__(self):
         """Context manager entry."""
         self._client = httpx.Client(
@@ -58,19 +58,19 @@ class HTTPClient:
             headers=self._get_headers(),
         )
         return self
-    
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Context manager exit."""
         if self._client:
             self._client.close()
             self._client = None
-    
+
     def _get_headers(self) -> Dict[str, str]:
         """Get request headers with user-agent."""
         headers = {"User-Agent": self.user_agent}
         headers.update(self.base_headers)
         return headers
-    
+
     def get(
         self,
         url: str,
@@ -80,13 +80,13 @@ class HTTPClient:
     ) -> httpx.Response:
         """
         Make a GET request.
-        
+
         Args:
             url: Request URL
             params: Query parameters
             headers: Additional headers
             allow_redirects: Follow redirects
-            
+
         Returns:
             httpx.Response object
         """
@@ -97,7 +97,7 @@ class HTTPClient:
             headers=headers,
             follow_redirects=allow_redirects,
         )
-    
+
     def post(
         self,
         url: str,
@@ -108,14 +108,14 @@ class HTTPClient:
     ) -> httpx.Response:
         """
         Make a POST request.
-        
+
         Args:
             url: Request URL
             data: Form data
             json: JSON data
             files: Files to upload
             headers: Additional headers
-            
+
         Returns:
             httpx.Response object
         """
@@ -127,7 +127,7 @@ class HTTPClient:
             files=files,
             headers=headers,
         )
-    
+
     def put(
         self,
         url: str,
@@ -139,7 +139,7 @@ class HTTPClient:
     ) -> httpx.Response:
         """
         Make a PUT request.
-        
+
         Args:
             url: Request URL
             data: Form data
@@ -147,7 +147,7 @@ class HTTPClient:
             content: Raw content
             files: Files to upload
             headers: Additional headers
-            
+
         Returns:
             httpx.Response object
         """
@@ -168,11 +168,11 @@ class HTTPClient:
     ) -> httpx.Response:
         """
         Make a DELETE request.
-        
+
         Args:
             url: Request URL
             headers: Additional headers
-            
+
         Returns:
             httpx.Response object
         """
@@ -182,25 +182,20 @@ class HTTPClient:
             headers=headers,
         )
 
-    def request(
-        self,
-        method: str,
-        url: str,
-        **kwargs
-    ) -> httpx.Response:
+    def request(self, method: str, url: str, **kwargs) -> httpx.Response:
         """
         Make a generic HTTP request.
-        
+
         Args:
             method: HTTP method
             url: Request URL
             **kwargs: Additional arguments
-            
+
         Returns:
             httpx.Response object
         """
         return self._request(method, url, **kwargs)
-    
+
     def download(
         self,
         url: str,
@@ -211,20 +206,20 @@ class HTTPClient:
     ) -> Path:
         """
         Download a file from URL with optional progress tracking.
-        
+
         Args:
             url: File URL
             file_path: Local file path to save to
             chunk_size: Download chunk size
             progress_callback: Callback(bytes_downloaded, total_bytes)
             headers: Additional headers
-            
+
         Returns:
             Path object of downloaded file
         """
         path = Path(file_path)
         path.parent.mkdir(parents=True, exist_ok=True)
-        
+
         with self._request_stream("GET", url, headers=headers, follow_redirects=True) as response:
             response.raise_for_status()
             total_bytes = int(response.headers.get("content-length", 0))
@@ -236,7 +231,7 @@ class HTTPClient:
                     progress_callback(0, total_bytes)
                 except Exception:
                     pass
-            
+
             with open(path, "wb") as f:
                 for chunk in response.iter_bytes(chunk_size):
                     if chunk:
@@ -251,31 +246,31 @@ class HTTPClient:
                     progress_callback(bytes_downloaded, total_bytes)
                 except Exception:
                     pass
-        
+
         return path
-    
+
     def _request(
         self,
         method: str,
         url: str,
         raise_for_status: bool = True,
         log_http_errors: bool = True,
-        **kwargs
+        **kwargs,
     ) -> httpx.Response:
         """
         Make an HTTP request with automatic retries.
-        
+
         Args:
             method: HTTP method
             url: Request URL
             **kwargs: Additional arguments for httpx.Client.request()
-            
+
         Returns:
             httpx.Response object
         """
         if not self._client:
             raise RuntimeError("HTTPClient must be used with context manager (with statement)")
-        
+
         # Merge headers
         if "headers" in kwargs and kwargs["headers"]:
             headers = self._get_headers()
@@ -283,9 +278,9 @@ class HTTPClient:
             kwargs["headers"] = headers
         else:
             kwargs["headers"] = self._get_headers()
-        
+
         last_exception = None
-        
+
         for attempt in range(self.retries):
             try:
                 response = self._client.request(method, url, **kwargs)
@@ -312,26 +307,30 @@ class HTTPClient:
                     response_text = e.response.text[:200]
                 except:
                     response_text = "<unable to read response>"
-                logger.warning(f"HTTP {e.response.status_code} on attempt {attempt + 1}/{self.retries}: {url} - {response_text}")
+                logger.warning(
+                    f"HTTP {e.response.status_code} on attempt {attempt + 1}/{self.retries}: {url} - {response_text}"
+                )
                 if attempt < self.retries - 1:
                     continue
             except (httpx.RequestError, httpx.ConnectError) as e:
                 last_exception = e
-                logger.warning(f"Connection error on attempt {attempt + 1}/{self.retries}: {url} - {e}")
+                logger.warning(
+                    f"Connection error on attempt {attempt + 1}/{self.retries}: {url} - {e}"
+                )
                 if attempt < self.retries - 1:
                     continue
-        
+
         if last_exception:
             logger.error(f"Request failed after {self.retries} attempts: {url} - {last_exception}")
             raise last_exception
-        
+
         raise RuntimeError("Request failed after retries")
-    
+
     def _request_stream(self, method: str, url: str, **kwargs):
         """Make a streaming request."""
         if not self._client:
             raise RuntimeError("HTTPClient must be used with context manager (with statement)")
-        
+
         # Merge headers
         if "headers" in kwargs and kwargs["headers"]:
             headers = self._get_headers()
@@ -339,13 +338,13 @@ class HTTPClient:
             kwargs["headers"] = headers
         else:
             kwargs["headers"] = self._get_headers()
-        
+
         return self._client.stream(method, url, **kwargs)
 
 
 class AsyncHTTPClient:
     """Unified async HTTP client with asyncio support."""
-    
+
     def __init__(
         self,
         timeout: float = DEFAULT_TIMEOUT,
@@ -356,7 +355,7 @@ class AsyncHTTPClient:
     ):
         """
         Initialize async HTTP client.
-        
+
         Args:
             timeout: Request timeout in seconds
             retries: Number of retries on transient failures
@@ -370,7 +369,7 @@ class AsyncHTTPClient:
         self.verify_ssl = verify_ssl
         self.base_headers = headers or {}
         self._client: Optional[httpx.AsyncClient] = None
-    
+
     async def __aenter__(self):
         """Async context manager entry."""
         self._client = httpx.AsyncClient(
@@ -379,19 +378,19 @@ class AsyncHTTPClient:
             headers=self._get_headers(),
         )
         return self
-    
+
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         """Async context manager exit."""
         if self._client:
             await self._client.aclose()
             self._client = None
-    
+
     def _get_headers(self) -> Dict[str, str]:
         """Get request headers with user-agent."""
         headers = {"User-Agent": self.user_agent}
         headers.update(self.base_headers)
         return headers
-    
+
     async def get(
         self,
         url: str,
@@ -401,13 +400,13 @@ class AsyncHTTPClient:
     ) -> httpx.Response:
         """
         Make an async GET request.
-        
+
         Args:
             url: Request URL
             params: Query parameters
             headers: Additional headers
             allow_redirects: Follow redirects
-            
+
         Returns:
             httpx.Response object
         """
@@ -418,7 +417,7 @@ class AsyncHTTPClient:
             headers=headers,
             follow_redirects=allow_redirects,
         )
-    
+
     async def post(
         self,
         url: str,
@@ -428,13 +427,13 @@ class AsyncHTTPClient:
     ) -> httpx.Response:
         """
         Make an async POST request.
-        
+
         Args:
             url: Request URL
             data: Form data
             json: JSON data
             headers: Additional headers
-            
+
         Returns:
             httpx.Response object
         """
@@ -445,7 +444,7 @@ class AsyncHTTPClient:
             json=json,
             headers=headers,
         )
-    
+
     async def download(
         self,
         url: str,
@@ -456,25 +455,25 @@ class AsyncHTTPClient:
     ) -> Path:
         """
         Download a file from URL asynchronously with optional progress tracking.
-        
+
         Args:
             url: File URL
             file_path: Local file path to save to
             chunk_size: Download chunk size
             progress_callback: Callback(bytes_downloaded, total_bytes)
             headers: Additional headers
-            
+
         Returns:
             Path object of downloaded file
         """
         path = Path(file_path)
         path.parent.mkdir(parents=True, exist_ok=True)
-        
+
         async with self._request_stream("GET", url, headers=headers) as response:
             response.raise_for_status()
             total_bytes = int(response.headers.get("content-length", 0))
             bytes_downloaded = 0
-            
+
             with open(path, "wb") as f:
                 async for chunk in response.aiter_bytes(chunk_size):
                     if chunk:
@@ -482,29 +481,24 @@ class AsyncHTTPClient:
                         bytes_downloaded += len(chunk)
                         if progress_callback:
                             progress_callback(bytes_downloaded, total_bytes)
-        
+
         return path
-    
-    async def _request(
-        self,
-        method: str,
-        url: str,
-        **kwargs
-    ) -> httpx.Response:
+
+    async def _request(self, method: str, url: str, **kwargs) -> httpx.Response:
         """
         Make an async HTTP request with automatic retries.
-        
+
         Args:
             method: HTTP method
             url: Request URL
             **kwargs: Additional arguments for httpx.AsyncClient.request()
-            
+
         Returns:
             httpx.Response object
         """
         if not self._client:
             raise RuntimeError("AsyncHTTPClient must be used with async context manager")
-        
+
         # Merge headers
         if "headers" in kwargs and kwargs["headers"]:
             headers = self._get_headers()
@@ -512,9 +506,9 @@ class AsyncHTTPClient:
             kwargs["headers"] = headers
         else:
             kwargs["headers"] = self._get_headers()
-        
+
         last_exception = None
-        
+
         for attempt in range(self.retries):
             try:
                 response = await self._client.request(method, url, **kwargs)
@@ -540,28 +534,32 @@ class AsyncHTTPClient:
                     response_text = e.response.text[:200]
                 except:
                     response_text = "<unable to read response>"
-                logger.warning(f"HTTP {e.response.status_code} on attempt {attempt + 1}/{self.retries}: {url} - {response_text}")
+                logger.warning(
+                    f"HTTP {e.response.status_code} on attempt {attempt + 1}/{self.retries}: {url} - {response_text}"
+                )
                 if attempt < self.retries - 1:
                     await asyncio.sleep(0.5)
                     continue
             except (httpx.RequestError, httpx.ConnectError) as e:
                 last_exception = e
-                logger.warning(f"Connection error on attempt {attempt + 1}/{self.retries}: {url} - {e}")
+                logger.warning(
+                    f"Connection error on attempt {attempt + 1}/{self.retries}: {url} - {e}"
+                )
                 if attempt < self.retries - 1:
                     await asyncio.sleep(0.5)
                     continue
-        
+
         if last_exception:
             logger.error(f"Request failed after {self.retries} attempts: {url} - {last_exception}")
             raise last_exception
-        
+
         raise RuntimeError("Request failed after retries")
-    
+
     def _request_stream(self, method: str, url: str, **kwargs):
         """Make a streaming request."""
         if not self._client:
             raise RuntimeError("AsyncHTTPClient must be used with async context manager")
-        
+
         # Merge headers
         if "headers" in kwargs and kwargs["headers"]:
             headers = self._get_headers()
@@ -569,7 +567,7 @@ class AsyncHTTPClient:
             kwargs["headers"] = headers
         else:
             kwargs["headers"] = self._get_headers()
-        
+
         return self._client.stream(method, url, **kwargs)
 
 
@@ -590,7 +588,7 @@ def download(
     url: str,
     file_path: str,
     progress_callback: Optional[Callable[[int, int], None]] = None,
-    **kwargs
+    **kwargs,
 ) -> Path:
     """Quick file download without context manager."""
     with HTTPClient() as client: