Add YAPF style + ignore, and format tracked Python files
This commit is contained in:
108
API/HTTP.py
108
API/HTTP.py
@@ -31,7 +31,8 @@ class HTTPClient:
|
||||
retries: int = DEFAULT_RETRIES,
|
||||
user_agent: str = DEFAULT_USER_AGENT,
|
||||
verify_ssl: bool = True,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize HTTP client.
|
||||
@@ -67,15 +68,19 @@ class HTTPClient:
|
||||
|
||||
def _get_headers(self) -> Dict[str, str]:
|
||||
"""Get request headers with user-agent."""
|
||||
headers = {"User-Agent": self.user_agent}
|
||||
headers = {
|
||||
"User-Agent": self.user_agent
|
||||
}
|
||||
headers.update(self.base_headers)
|
||||
return headers
|
||||
|
||||
def get(
|
||||
self,
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
params: Optional[Dict[str,
|
||||
Any]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
allow_redirects: bool = True,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
@@ -104,7 +109,8 @@ class HTTPClient:
|
||||
data: Optional[Any] = None,
|
||||
json: Optional[Dict] = None,
|
||||
files: Optional[Dict] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make a POST request.
|
||||
@@ -135,7 +141,8 @@ class HTTPClient:
|
||||
json: Optional[Dict] = None,
|
||||
content: Optional[Any] = None,
|
||||
files: Optional[Dict] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make a PUT request.
|
||||
@@ -164,7 +171,8 @@ class HTTPClient:
|
||||
def delete(
|
||||
self,
|
||||
url: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make a DELETE request.
|
||||
@@ -201,8 +209,11 @@ class HTTPClient:
|
||||
url: str,
|
||||
file_path: str,
|
||||
chunk_size: int = 8192,
|
||||
progress_callback: Optional[Callable[[int, int], None]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
progress_callback: Optional[Callable[[int,
|
||||
int],
|
||||
None]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> Path:
|
||||
"""
|
||||
Download a file from URL with optional progress tracking.
|
||||
@@ -220,7 +231,10 @@ class HTTPClient:
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with self._request_stream("GET", url, headers=headers, follow_redirects=True) as response:
|
||||
with self._request_stream("GET",
|
||||
url,
|
||||
headers=headers,
|
||||
follow_redirects=True) as response:
|
||||
response.raise_for_status()
|
||||
total_bytes = int(response.headers.get("content-length", 0))
|
||||
bytes_downloaded = 0
|
||||
@@ -269,7 +283,9 @@ class HTTPClient:
|
||||
httpx.Response object
|
||||
"""
|
||||
if not self._client:
|
||||
raise RuntimeError("HTTPClient must be used with context manager (with statement)")
|
||||
raise RuntimeError(
|
||||
"HTTPClient must be used with context manager (with statement)"
|
||||
)
|
||||
|
||||
# Merge headers
|
||||
if "headers" in kwargs and kwargs["headers"]:
|
||||
@@ -289,7 +305,9 @@ class HTTPClient:
|
||||
return response
|
||||
except httpx.TimeoutException as e:
|
||||
last_exception = e
|
||||
logger.warning(f"Timeout on attempt {attempt + 1}/{self.retries}: {url}")
|
||||
logger.warning(
|
||||
f"Timeout on attempt {attempt + 1}/{self.retries}: {url}"
|
||||
)
|
||||
if attempt < self.retries - 1:
|
||||
continue
|
||||
except httpx.HTTPStatusError as e:
|
||||
@@ -300,7 +318,9 @@ class HTTPClient:
|
||||
except:
|
||||
response_text = "<unable to read response>"
|
||||
if log_http_errors:
|
||||
logger.error(f"HTTP {e.response.status_code} from {url}: {response_text}")
|
||||
logger.error(
|
||||
f"HTTP {e.response.status_code} from {url}: {response_text}"
|
||||
)
|
||||
raise
|
||||
last_exception = e
|
||||
try:
|
||||
@@ -321,7 +341,9 @@ class HTTPClient:
|
||||
continue
|
||||
|
||||
if last_exception:
|
||||
logger.error(f"Request failed after {self.retries} attempts: {url} - {last_exception}")
|
||||
logger.error(
|
||||
f"Request failed after {self.retries} attempts: {url} - {last_exception}"
|
||||
)
|
||||
raise last_exception
|
||||
|
||||
raise RuntimeError("Request failed after retries")
|
||||
@@ -329,7 +351,9 @@ class HTTPClient:
|
||||
def _request_stream(self, method: str, url: str, **kwargs):
|
||||
"""Make a streaming request."""
|
||||
if not self._client:
|
||||
raise RuntimeError("HTTPClient must be used with context manager (with statement)")
|
||||
raise RuntimeError(
|
||||
"HTTPClient must be used with context manager (with statement)"
|
||||
)
|
||||
|
||||
# Merge headers
|
||||
if "headers" in kwargs and kwargs["headers"]:
|
||||
@@ -351,7 +375,8 @@ class AsyncHTTPClient:
|
||||
retries: int = DEFAULT_RETRIES,
|
||||
user_agent: str = DEFAULT_USER_AGENT,
|
||||
verify_ssl: bool = True,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize async HTTP client.
|
||||
@@ -387,15 +412,19 @@ class AsyncHTTPClient:
|
||||
|
||||
def _get_headers(self) -> Dict[str, str]:
|
||||
"""Get request headers with user-agent."""
|
||||
headers = {"User-Agent": self.user_agent}
|
||||
headers = {
|
||||
"User-Agent": self.user_agent
|
||||
}
|
||||
headers.update(self.base_headers)
|
||||
return headers
|
||||
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
params: Optional[Dict[str,
|
||||
Any]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
allow_redirects: bool = True,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
@@ -423,7 +452,8 @@ class AsyncHTTPClient:
|
||||
url: str,
|
||||
data: Optional[Any] = None,
|
||||
json: Optional[Dict] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make an async POST request.
|
||||
@@ -450,8 +480,11 @@ class AsyncHTTPClient:
|
||||
url: str,
|
||||
file_path: str,
|
||||
chunk_size: int = 8192,
|
||||
progress_callback: Optional[Callable[[int, int], None]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
progress_callback: Optional[Callable[[int,
|
||||
int],
|
||||
None]] = None,
|
||||
headers: Optional[Dict[str,
|
||||
str]] = None,
|
||||
) -> Path:
|
||||
"""
|
||||
Download a file from URL asynchronously with optional progress tracking.
|
||||
@@ -497,7 +530,9 @@ class AsyncHTTPClient:
|
||||
httpx.Response object
|
||||
"""
|
||||
if not self._client:
|
||||
raise RuntimeError("AsyncHTTPClient must be used with async context manager")
|
||||
raise RuntimeError(
|
||||
"AsyncHTTPClient must be used with async context manager"
|
||||
)
|
||||
|
||||
# Merge headers
|
||||
if "headers" in kwargs and kwargs["headers"]:
|
||||
@@ -516,7 +551,9 @@ class AsyncHTTPClient:
|
||||
return response
|
||||
except httpx.TimeoutException as e:
|
||||
last_exception = e
|
||||
logger.warning(f"Timeout on attempt {attempt + 1}/{self.retries}: {url}")
|
||||
logger.warning(
|
||||
f"Timeout on attempt {attempt + 1}/{self.retries}: {url}"
|
||||
)
|
||||
if attempt < self.retries - 1:
|
||||
await asyncio.sleep(0.5) # Brief delay before retry
|
||||
continue
|
||||
@@ -527,7 +564,9 @@ class AsyncHTTPClient:
|
||||
response_text = e.response.text[:500]
|
||||
except:
|
||||
response_text = "<unable to read response>"
|
||||
logger.error(f"HTTP {e.response.status_code} from {url}: {response_text}")
|
||||
logger.error(
|
||||
f"HTTP {e.response.status_code} from {url}: {response_text}"
|
||||
)
|
||||
raise
|
||||
last_exception = e
|
||||
try:
|
||||
@@ -550,7 +589,9 @@ class AsyncHTTPClient:
|
||||
continue
|
||||
|
||||
if last_exception:
|
||||
logger.error(f"Request failed after {self.retries} attempts: {url} - {last_exception}")
|
||||
logger.error(
|
||||
f"Request failed after {self.retries} attempts: {url} - {last_exception}"
|
||||
)
|
||||
raise last_exception
|
||||
|
||||
raise RuntimeError("Request failed after retries")
|
||||
@@ -558,7 +599,9 @@ class AsyncHTTPClient:
|
||||
def _request_stream(self, method: str, url: str, **kwargs):
|
||||
"""Make a streaming request."""
|
||||
if not self._client:
|
||||
raise RuntimeError("AsyncHTTPClient must be used with async context manager")
|
||||
raise RuntimeError(
|
||||
"AsyncHTTPClient must be used with async context manager"
|
||||
)
|
||||
|
||||
# Merge headers
|
||||
if "headers" in kwargs and kwargs["headers"]:
|
||||
@@ -587,9 +630,16 @@ def post(url: str, **kwargs) -> httpx.Response:
|
||||
def download(
|
||||
url: str,
|
||||
file_path: str,
|
||||
progress_callback: Optional[Callable[[int, int], None]] = None,
|
||||
progress_callback: Optional[Callable[[int,
|
||||
int],
|
||||
None]] = None,
|
||||
**kwargs,
|
||||
) -> Path:
|
||||
"""Quick file download without context manager."""
|
||||
with HTTPClient() as client:
|
||||
return client.download(url, file_path, progress_callback=progress_callback, **kwargs)
|
||||
return client.download(
|
||||
url,
|
||||
file_path,
|
||||
progress_callback=progress_callback,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
@@ -85,7 +85,8 @@ class HydrusNetwork:
|
||||
raise ValueError("Hydrus base URL is required")
|
||||
self.url = self.url.rstrip("/")
|
||||
parsed = urlsplit(self.url)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
if parsed.scheme not in {"http",
|
||||
"https"}:
|
||||
raise ValueError("Hydrus base URL must use http or https")
|
||||
self.scheme = parsed.scheme
|
||||
self.hostname = parsed.hostname or "localhost"
|
||||
@@ -114,7 +115,8 @@ class HydrusNetwork:
|
||||
return path
|
||||
|
||||
def _perform_request(self, spec: HydrusRequestSpec) -> Any:
|
||||
headers: dict[str, str] = {}
|
||||
headers: dict[str,
|
||||
str] = {}
|
||||
|
||||
# Use session key if available, otherwise use access key
|
||||
if self._session_key:
|
||||
@@ -138,7 +140,9 @@ class HydrusNetwork:
|
||||
content_type = ""
|
||||
|
||||
try:
|
||||
with HTTPClient(timeout=self.timeout, headers=headers, verify_ssl=False) as client:
|
||||
with HTTPClient(timeout=self.timeout,
|
||||
headers=headers,
|
||||
verify_ssl=False) as client:
|
||||
response = None
|
||||
|
||||
if spec.file_path is not None:
|
||||
@@ -149,7 +153,8 @@ class HydrusNetwork:
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
file_size = file_path.stat().st_size
|
||||
headers["Content-Type"] = spec.content_type or "application/octet-stream"
|
||||
headers["Content-Type"
|
||||
] = spec.content_type or "application/octet-stream"
|
||||
# Do not set Content-Length when streaming an iterator body.
|
||||
# If the file size changes between stat() and read() (or the source is truncated),
|
||||
# h11 will raise: "Too little data for declared Content-Length".
|
||||
@@ -239,7 +244,9 @@ class HydrusNetwork:
|
||||
body = response.content
|
||||
content_type = response.headers.get("Content-Type", "") or ""
|
||||
|
||||
logger.debug(f"{self._log_prefix()} Response {status} {reason} ({len(body)} bytes)")
|
||||
logger.debug(
|
||||
f"{self._log_prefix()} Response {status} {reason} ({len(body)} bytes)"
|
||||
)
|
||||
|
||||
except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) as exc:
|
||||
msg = f"Hydrus unavailable: {exc}"
|
||||
@@ -292,7 +299,10 @@ class HydrusNetwork:
|
||||
# Retry the request with new session key
|
||||
return self._perform_request(spec)
|
||||
except Exception as retry_error:
|
||||
logger.error(f"{self._log_prefix()} Retry failed: {retry_error}", exc_info=True)
|
||||
logger.error(
|
||||
f"{self._log_prefix()} Retry failed: {retry_error}",
|
||||
exc_info=True
|
||||
)
|
||||
# If retry fails, raise the original error
|
||||
raise HydrusRequestError(status, message, payload) from retry_error
|
||||
|
||||
@@ -311,7 +321,10 @@ class HydrusNetwork:
|
||||
Raises HydrusRequestError if the request fails.
|
||||
"""
|
||||
if not self.access_key:
|
||||
raise HydrusRequestError(401, "Cannot acquire session key: no access key configured")
|
||||
raise HydrusRequestError(
|
||||
401,
|
||||
"Cannot acquire session key: no access key configured"
|
||||
)
|
||||
|
||||
# Temporarily use access key to get session key
|
||||
original_session_key = self._session_key
|
||||
@@ -323,7 +336,9 @@ class HydrusNetwork:
|
||||
|
||||
if not session_key:
|
||||
raise HydrusRequestError(
|
||||
500, "Session key response missing 'session_key' field", result
|
||||
500,
|
||||
"Session key response missing 'session_key' field",
|
||||
result
|
||||
)
|
||||
|
||||
self._session_key = session_key
|
||||
@@ -345,7 +360,12 @@ class HydrusNetwork:
|
||||
return self._session_key
|
||||
return self._acquire_session_key()
|
||||
|
||||
def _get(self, endpoint: str, *, query: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
def _get(self,
|
||||
endpoint: str,
|
||||
*,
|
||||
query: dict[str,
|
||||
Any] | None = None) -> dict[str,
|
||||
Any]:
|
||||
spec = HydrusRequestSpec("GET", endpoint, query=query)
|
||||
return cast(dict[str, Any], self._perform_request(spec))
|
||||
|
||||
@@ -353,12 +373,18 @@ class HydrusNetwork:
|
||||
self,
|
||||
endpoint: str,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
data: dict[str,
|
||||
Any] | None = None,
|
||||
file_path: Path | None = None,
|
||||
content_type: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
spec = HydrusRequestSpec(
|
||||
"POST", endpoint, data=data, file_path=file_path, content_type=content_type
|
||||
"POST",
|
||||
endpoint,
|
||||
data=data,
|
||||
file_path=file_path,
|
||||
content_type=content_type
|
||||
)
|
||||
return cast(dict[str, Any], self._perform_request(spec))
|
||||
|
||||
@@ -397,12 +423,19 @@ class HydrusNetwork:
|
||||
Required JSON args: {"hashes": [<sha256 hex>, ...]}
|
||||
"""
|
||||
hash_list = self._ensure_hashes(hashes)
|
||||
body = {"hashes": hash_list}
|
||||
body = {
|
||||
"hashes": hash_list
|
||||
}
|
||||
return self._post("/add_files/undelete_files", data=body)
|
||||
|
||||
def delete_files(
|
||||
self, hashes: Union[str, Iterable[str]], *, reason: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
self,
|
||||
hashes: Union[str,
|
||||
Iterable[str]],
|
||||
*,
|
||||
reason: str | None = None
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""Delete files in Hydrus.
|
||||
|
||||
Hydrus Client API: POST /add_files/delete_files
|
||||
@@ -410,98 +443,166 @@ class HydrusNetwork:
|
||||
Optional JSON args: {"reason": "..."}
|
||||
"""
|
||||
hash_list = self._ensure_hashes(hashes)
|
||||
body: dict[str, Any] = {"hashes": hash_list}
|
||||
body: dict[str,
|
||||
Any] = {
|
||||
"hashes": hash_list
|
||||
}
|
||||
if isinstance(reason, str) and reason.strip():
|
||||
body["reason"] = reason.strip()
|
||||
return self._post("/add_files/delete_files", data=body)
|
||||
|
||||
def clear_file_deletion_record(self, hashes: Union[str, Iterable[str]]) -> dict[str, Any]:
|
||||
def clear_file_deletion_record(self,
|
||||
hashes: Union[str,
|
||||
Iterable[str]]) -> dict[str,
|
||||
Any]:
|
||||
"""Clear Hydrus's file deletion record for the provided hashes.
|
||||
|
||||
Hydrus Client API: POST /add_files/clear_file_deletion_record
|
||||
Required JSON args: {"hashes": [<sha256 hex>, ...]}
|
||||
"""
|
||||
hash_list = self._ensure_hashes(hashes)
|
||||
body = {"hashes": hash_list}
|
||||
body = {
|
||||
"hashes": hash_list
|
||||
}
|
||||
return self._post("/add_files/clear_file_deletion_record", data=body)
|
||||
|
||||
def add_tag(
|
||||
self, hash: Union[str, Iterable[str]], tags: Iterable[str], service_name: str
|
||||
) -> dict[str, Any]:
|
||||
self,
|
||||
hash: Union[str,
|
||||
Iterable[str]],
|
||||
tags: Iterable[str],
|
||||
service_name: str
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
hash = self._ensure_hashes(hash)
|
||||
body = {"hashes": hash, "service_names_to_tags": {service_name: list(tags)}}
|
||||
body = {
|
||||
"hashes": hash,
|
||||
"service_names_to_tags": {
|
||||
service_name: list(tags)
|
||||
}
|
||||
}
|
||||
return self._post("/add_tags/add_tags", data=body)
|
||||
|
||||
def delete_tag(
|
||||
self,
|
||||
file_hashes: Union[str, Iterable[str]],
|
||||
file_hashes: Union[str,
|
||||
Iterable[str]],
|
||||
tags: Iterable[str],
|
||||
service_name: str,
|
||||
*,
|
||||
action: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
hashes = self._ensure_hashes(file_hashes)
|
||||
body = {
|
||||
"hashes": hashes,
|
||||
"service_names_to_actions_to_tags": {service_name: {action: list(tags)}},
|
||||
"service_names_to_actions_to_tags": {
|
||||
service_name: {
|
||||
action: list(tags)
|
||||
}
|
||||
},
|
||||
}
|
||||
return self._post("/add_tags/add_tags", data=body)
|
||||
|
||||
def add_tags_by_key(
|
||||
self, hash: Union[str, Iterable[str]], tags: Iterable[str], service_key: str
|
||||
) -> dict[str, Any]:
|
||||
self,
|
||||
hash: Union[str,
|
||||
Iterable[str]],
|
||||
tags: Iterable[str],
|
||||
service_key: str
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
hash = self._ensure_hashes(hash)
|
||||
body = {"hashes": hash, "service_keys_to_tags": {service_key: list(tags)}}
|
||||
body = {
|
||||
"hashes": hash,
|
||||
"service_keys_to_tags": {
|
||||
service_key: list(tags)
|
||||
}
|
||||
}
|
||||
return self._post("/add_tags/add_tags", data=body)
|
||||
|
||||
def delete_tags_by_key(
|
||||
self,
|
||||
file_hashes: Union[str, Iterable[str]],
|
||||
file_hashes: Union[str,
|
||||
Iterable[str]],
|
||||
tags: Iterable[str],
|
||||
service_key: str,
|
||||
*,
|
||||
action: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
hashes = self._ensure_hashes(file_hashes)
|
||||
body = {
|
||||
"hashes": hashes,
|
||||
"service_keys_to_actions_to_tags": {service_key: {action: list(tags)}},
|
||||
"service_keys_to_actions_to_tags": {
|
||||
service_key: {
|
||||
action: list(tags)
|
||||
}
|
||||
},
|
||||
}
|
||||
return self._post("/add_tags/add_tags", data=body)
|
||||
|
||||
def associate_url(self, file_hashes: Union[str, Iterable[str]], url: str) -> dict[str, Any]:
|
||||
def associate_url(self,
|
||||
file_hashes: Union[str,
|
||||
Iterable[str]],
|
||||
url: str) -> dict[str,
|
||||
Any]:
|
||||
hashes = self._ensure_hashes(file_hashes)
|
||||
if len(hashes) == 1:
|
||||
body = {"hash": hashes[0], "url_to_add": url}
|
||||
body = {
|
||||
"hash": hashes[0],
|
||||
"url_to_add": url
|
||||
}
|
||||
return self._post("/add_urls/associate_url", data=body)
|
||||
|
||||
results: dict[str, Any] = {}
|
||||
results: dict[str,
|
||||
Any] = {}
|
||||
for file_hash in hashes:
|
||||
body = {"hash": file_hash, "url_to_add": url}
|
||||
body = {
|
||||
"hash": file_hash,
|
||||
"url_to_add": url
|
||||
}
|
||||
results[file_hash] = self._post("/add_urls/associate_url", data=body)
|
||||
return {"batched": results}
|
||||
return {
|
||||
"batched": results
|
||||
}
|
||||
|
||||
def delete_url(self, file_hashes: Union[str, Iterable[str]], url: str) -> dict[str, Any]:
|
||||
def delete_url(self,
|
||||
file_hashes: Union[str,
|
||||
Iterable[str]],
|
||||
url: str) -> dict[str,
|
||||
Any]:
|
||||
hashes = self._ensure_hashes(file_hashes)
|
||||
if len(hashes) == 1:
|
||||
body = {"hash": hashes[0], "url_to_delete": url}
|
||||
body = {
|
||||
"hash": hashes[0],
|
||||
"url_to_delete": url
|
||||
}
|
||||
return self._post("/add_urls/associate_url", data=body)
|
||||
|
||||
results: dict[str, Any] = {}
|
||||
results: dict[str,
|
||||
Any] = {}
|
||||
for file_hash in hashes:
|
||||
body = {"hash": file_hash, "url_to_delete": url}
|
||||
body = {
|
||||
"hash": file_hash,
|
||||
"url_to_delete": url
|
||||
}
|
||||
results[file_hash] = self._post("/add_urls/associate_url", data=body)
|
||||
return {"batched": results}
|
||||
return {
|
||||
"batched": results
|
||||
}
|
||||
|
||||
def set_notes(
|
||||
self,
|
||||
file_hash: str,
|
||||
notes: dict[str, str],
|
||||
notes: dict[str,
|
||||
str],
|
||||
*,
|
||||
merge_cleverly: bool = False,
|
||||
extend_existing_note_if_possible: bool = True,
|
||||
conflict_resolution: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""Add or update notes associated with a file.
|
||||
|
||||
Hydrus Client API: POST /add_notes/set_notes
|
||||
@@ -514,11 +615,17 @@ class HydrusNetwork:
|
||||
if not file_hash:
|
||||
raise ValueError("file_hash must not be empty")
|
||||
|
||||
body: dict[str, Any] = {"hash": file_hash, "notes": notes}
|
||||
body: dict[str,
|
||||
Any] = {
|
||||
"hash": file_hash,
|
||||
"notes": notes
|
||||
}
|
||||
|
||||
if merge_cleverly:
|
||||
body["merge_cleverly"] = True
|
||||
body["extend_existing_note_if_possible"] = bool(extend_existing_note_if_possible)
|
||||
body["extend_existing_note_if_possible"] = bool(
|
||||
extend_existing_note_if_possible
|
||||
)
|
||||
body["conflict_resolution"] = int(conflict_resolution)
|
||||
return self._post("/add_notes/set_notes", data=body)
|
||||
|
||||
@@ -526,7 +633,8 @@ class HydrusNetwork:
|
||||
self,
|
||||
file_hash: str,
|
||||
note_names: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""Delete notes associated with a file.
|
||||
|
||||
Hydrus Client API: POST /add_notes/delete_notes
|
||||
@@ -540,20 +648,30 @@ class HydrusNetwork:
|
||||
if not file_hash:
|
||||
raise ValueError("file_hash must not be empty")
|
||||
|
||||
body = {"hash": file_hash, "note_names": names}
|
||||
body = {
|
||||
"hash": file_hash,
|
||||
"note_names": names
|
||||
}
|
||||
return self._post("/add_notes/delete_notes", data=body)
|
||||
|
||||
def get_file_relationships(self, file_hash: str) -> dict[str, Any]:
|
||||
query = {"hash": file_hash}
|
||||
return self._get("/manage_file_relationships/get_file_relationships", query=query)
|
||||
query = {
|
||||
"hash": file_hash
|
||||
}
|
||||
return self._get(
|
||||
"/manage_file_relationships/get_file_relationships",
|
||||
query=query
|
||||
)
|
||||
|
||||
def set_relationship(
|
||||
self,
|
||||
hash_a: str,
|
||||
hash_b: str,
|
||||
relationship: Union[str, int],
|
||||
relationship: Union[str,
|
||||
int],
|
||||
do_default_content_merge: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""Set a relationship between two files in Hydrus.
|
||||
|
||||
This wraps Hydrus Client API: POST /manage_file_relationships/set_file_relationships.
|
||||
@@ -609,7 +727,10 @@ class HydrusNetwork:
|
||||
# Hydrus does not accept 'king' as a relationship; this maps to 'A is better'.
|
||||
"king": 4,
|
||||
}
|
||||
relationship = rel_map.get(relationship.lower().strip(), 3) # Default to alternates
|
||||
relationship = rel_map.get(
|
||||
relationship.lower().strip(),
|
||||
3
|
||||
) # Default to alternates
|
||||
|
||||
body = {
|
||||
"relationships": [
|
||||
@@ -621,7 +742,10 @@ class HydrusNetwork:
|
||||
}
|
||||
]
|
||||
}
|
||||
return self._post("/manage_file_relationships/set_file_relationships", data=body)
|
||||
return self._post(
|
||||
"/manage_file_relationships/set_file_relationships",
|
||||
data=body
|
||||
)
|
||||
|
||||
def get_services(self) -> dict[str, Any]:
|
||||
return self._get("/get_services")
|
||||
@@ -639,17 +763,24 @@ class HydrusNetwork:
|
||||
file_sort_type: int | None = None,
|
||||
file_sort_asc: bool | None = None,
|
||||
file_sort_key: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
if not tags:
|
||||
raise ValueError("tags must not be empty")
|
||||
|
||||
query: dict[str, Any] = {}
|
||||
query: dict[str,
|
||||
Any] = {}
|
||||
query_fields = [
|
||||
("tags", tags, lambda v: json.dumps(list(v))),
|
||||
("file_service_name", file_service_name, lambda v: v),
|
||||
("return_hashes", return_hashes, lambda v: "true" if v else None),
|
||||
("return_file_ids", return_file_ids, lambda v: "true" if v else None),
|
||||
("return_file_count", return_file_count, lambda v: "true" if v else None),
|
||||
("tags",
|
||||
tags, lambda v: json.dumps(list(v))),
|
||||
("file_service_name",
|
||||
file_service_name, lambda v: v),
|
||||
("return_hashes",
|
||||
return_hashes, lambda v: "true" if v else None),
|
||||
("return_file_ids",
|
||||
return_file_ids, lambda v: "true" if v else None),
|
||||
("return_file_count",
|
||||
return_file_count, lambda v: "true" if v else None),
|
||||
(
|
||||
"include_current_tags",
|
||||
include_current_tags,
|
||||
@@ -660,13 +791,17 @@ class HydrusNetwork:
|
||||
include_pending_tags,
|
||||
lambda v: "true" if v else "false" if v is not None else None,
|
||||
),
|
||||
("file_sort_type", file_sort_type, lambda v: str(v) if v is not None else None),
|
||||
(
|
||||
"file_sort_type",
|
||||
file_sort_type, lambda v: str(v) if v is not None else None
|
||||
),
|
||||
(
|
||||
"file_sort_asc",
|
||||
file_sort_asc,
|
||||
lambda v: "true" if v else "false" if v is not None else None,
|
||||
),
|
||||
("file_sort_key", file_sort_key, lambda v: v),
|
||||
("file_sort_key",
|
||||
file_sort_key, lambda v: v),
|
||||
]
|
||||
|
||||
for key, value, formatter in query_fields:
|
||||
@@ -689,24 +824,33 @@ class HydrusNetwork:
|
||||
include_size: bool = True,
|
||||
include_mime: bool = False,
|
||||
include_notes: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
if not file_ids and not hashes:
|
||||
raise ValueError("Either file_ids or hashes must be provided")
|
||||
|
||||
query: dict[str, Any] = {}
|
||||
query: dict[str,
|
||||
Any] = {}
|
||||
query_fields = [
|
||||
("file_ids", file_ids, lambda v: json.dumps(list(v))),
|
||||
("hashes", hashes, lambda v: json.dumps(list(v))),
|
||||
("file_ids",
|
||||
file_ids, lambda v: json.dumps(list(v))),
|
||||
("hashes",
|
||||
hashes, lambda v: json.dumps(list(v))),
|
||||
(
|
||||
"include_service_keys_to_tags",
|
||||
include_service_keys_to_tags,
|
||||
lambda v: "true" if v else None,
|
||||
),
|
||||
("include_file_url", include_file_url, lambda v: "true" if v else None),
|
||||
("include_duration", include_duration, lambda v: "true" if v else None),
|
||||
("include_size", include_size, lambda v: "true" if v else None),
|
||||
("include_mime", include_mime, lambda v: "true" if v else None),
|
||||
("include_notes", include_notes, lambda v: "true" if v else None),
|
||||
("include_file_url",
|
||||
include_file_url, lambda v: "true" if v else None),
|
||||
("include_duration",
|
||||
include_duration, lambda v: "true" if v else None),
|
||||
("include_size",
|
||||
include_size, lambda v: "true" if v else None),
|
||||
("include_mime",
|
||||
include_mime, lambda v: "true" if v else None),
|
||||
("include_notes",
|
||||
include_notes, lambda v: "true" if v else None),
|
||||
]
|
||||
|
||||
for key, value, formatter in query_fields:
|
||||
@@ -720,7 +864,9 @@ class HydrusNetwork:
|
||||
|
||||
def get_file_path(self, file_hash: str) -> dict[str, Any]:
|
||||
"""Get the local file system path for a given file hash."""
|
||||
query = {"hash": file_hash}
|
||||
query = {
|
||||
"hash": file_hash
|
||||
}
|
||||
return self._get("/get_files/file_path", query=query)
|
||||
|
||||
def file_url(self, file_hash: str) -> str:
|
||||
@@ -752,7 +898,10 @@ class HydrusCliOptions:
|
||||
debug: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_namespace(cls: Type[HydrusCliOptionsT], namespace: Any) -> HydrusCliOptionsT:
|
||||
def from_namespace(
|
||||
cls: Type[HydrusCliOptionsT],
|
||||
namespace: Any
|
||||
) -> HydrusCliOptionsT:
|
||||
accept_header = namespace.accept or "application/cbor"
|
||||
body_bytes: bytes | None = None
|
||||
body_path: Path | None = None
|
||||
@@ -785,7 +934,8 @@ def hydrus_request(args, parser) -> int:
|
||||
if not parsed.hostname:
|
||||
parser.error("Invalid Hydrus URL")
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
headers: dict[str,
|
||||
str] = {}
|
||||
if options.access_key:
|
||||
headers["Hydrus-Client-API-Access-Key"] = options.access_key
|
||||
if options.accept:
|
||||
@@ -797,7 +947,10 @@ def hydrus_request(args, parser) -> int:
|
||||
body_path = options.body_path
|
||||
if not body_path.is_file():
|
||||
parser.error(f"File not found: {body_path}")
|
||||
headers.setdefault("Content-Type", options.content_type or "application/octet-stream")
|
||||
headers.setdefault(
|
||||
"Content-Type",
|
||||
options.content_type or "application/octet-stream"
|
||||
)
|
||||
headers["Content-Length"] = str(body_path.stat().st_size)
|
||||
elif options.body_bytes is not None:
|
||||
request_body_bytes = options.body_bytes
|
||||
@@ -820,13 +973,17 @@ def hydrus_request(args, parser) -> int:
|
||||
port = 443 if parsed.scheme == "https" else 80
|
||||
|
||||
connection_cls = (
|
||||
http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection
|
||||
http.client.HTTPSConnection
|
||||
if parsed.scheme == "https" else http.client.HTTPConnection
|
||||
)
|
||||
host = parsed.hostname or "localhost"
|
||||
connection = connection_cls(host, port, timeout=options.timeout)
|
||||
|
||||
if options.debug:
|
||||
log(f"Hydrus connecting to {parsed.scheme}://{host}:{port}{path}", file=sys.stderr)
|
||||
log(
|
||||
f"Hydrus connecting to {parsed.scheme}://{host}:{port}{path}",
|
||||
file=sys.stderr
|
||||
)
|
||||
response_bytes: bytes = b""
|
||||
content_type = ""
|
||||
status = 0
|
||||
@@ -835,12 +992,17 @@ def hydrus_request(args, parser) -> int:
|
||||
with body_path.open("rb") as handle:
|
||||
if options.debug:
|
||||
size_hint = headers.get("Content-Length", "unknown")
|
||||
log(f"Hydrus sending file body ({size_hint} bytes)", file=sys.stderr)
|
||||
log(
|
||||
f"Hydrus sending file body ({size_hint} bytes)",
|
||||
file=sys.stderr
|
||||
)
|
||||
connection.putrequest(options.method, path)
|
||||
host_header = host
|
||||
if (parsed.scheme == "http" and port not in (80, None)) or (
|
||||
parsed.scheme == "https" and port not in (443, None)
|
||||
):
|
||||
if (parsed.scheme == "http"
|
||||
and port not in (80,
|
||||
None)) or (parsed.scheme == "https"
|
||||
and port not in (443,
|
||||
None)):
|
||||
host_header = f"{host}:{port}"
|
||||
connection.putheader("Host", host_header)
|
||||
for key, value in headers.items():
|
||||
@@ -853,20 +1015,34 @@ def hydrus_request(args, parser) -> int:
|
||||
break
|
||||
connection.send(chunk)
|
||||
if options.debug:
|
||||
log("[downlow.py] Hydrus upload complete; awaiting response", file=sys.stderr)
|
||||
log(
|
||||
"[downlow.py] Hydrus upload complete; awaiting response",
|
||||
file=sys.stderr
|
||||
)
|
||||
else:
|
||||
if options.debug:
|
||||
size_hint = "none" if request_body_bytes is None else str(len(request_body_bytes))
|
||||
size_hint = "none" if request_body_bytes is None else str(
|
||||
len(request_body_bytes)
|
||||
)
|
||||
log(f"Hydrus sending request body bytes={size_hint}", file=sys.stderr)
|
||||
sanitized_headers = {k: v for k, v in headers.items() if v}
|
||||
sanitized_headers = {
|
||||
k: v
|
||||
for k, v in headers.items() if v
|
||||
}
|
||||
connection.request(
|
||||
options.method, path, body=request_body_bytes, headers=sanitized_headers
|
||||
options.method,
|
||||
path,
|
||||
body=request_body_bytes,
|
||||
headers=sanitized_headers
|
||||
)
|
||||
response = connection.getresponse()
|
||||
status = response.status
|
||||
response_bytes = response.read()
|
||||
if options.debug:
|
||||
log(f"Hydrus response received ({len(response_bytes)} bytes)", file=sys.stderr)
|
||||
log(
|
||||
f"Hydrus response received ({len(response_bytes)} bytes)",
|
||||
file=sys.stderr
|
||||
)
|
||||
content_type = response.getheader("Content-Type", "")
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
log(f"HTTP error: {exc}", file=sys.stderr)
|
||||
@@ -890,7 +1066,10 @@ def hydrus_request(args, parser) -> int:
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
payload = response_bytes.decode("utf-8", "replace")
|
||||
elif payload is None and expect_cbor and decode_error is not None:
|
||||
log(f"Expected CBOR response but decoding failed: {decode_error}", file=sys.stderr)
|
||||
log(
|
||||
f"Expected CBOR response but decoding failed: {decode_error}",
|
||||
file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
json_ready = jsonify(payload) if isinstance(payload, (dict, list)) else payload
|
||||
if options.debug:
|
||||
@@ -900,7 +1079,10 @@ def hydrus_request(args, parser) -> int:
|
||||
elif json_ready is None:
|
||||
log("{}")
|
||||
else:
|
||||
log(json.dumps({"value": json_ready}, ensure_ascii=False))
|
||||
log(json.dumps({
|
||||
"value": json_ready
|
||||
},
|
||||
ensure_ascii=False))
|
||||
return 0 if 200 <= status < 400 else 1
|
||||
|
||||
|
||||
@@ -1030,13 +1212,16 @@ def hydrus_export(args, _parser) -> int:
|
||||
hydrus_url = getattr(args, "hydrus_url", None)
|
||||
if not hydrus_url:
|
||||
try:
|
||||
from config import load_config, get_hydrus_url
|
||||
from SYS.config import load_config, get_hydrus_url
|
||||
|
||||
hydrus_url = get_hydrus_url(load_config())
|
||||
except Exception as exc:
|
||||
hydrus_url = None
|
||||
if os.environ.get("DOWNLOW_DEBUG"):
|
||||
log(f"hydrus-export could not load Hydrus URL: {exc}", file=sys.stderr)
|
||||
log(
|
||||
f"hydrus-export could not load Hydrus URL: {exc}",
|
||||
file=sys.stderr
|
||||
)
|
||||
if hydrus_url:
|
||||
try:
|
||||
setattr(args, "hydrus_url", hydrus_url)
|
||||
@@ -1047,19 +1232,30 @@ def hydrus_export(args, _parser) -> int:
|
||||
if hydrus_url and file_hash:
|
||||
try:
|
||||
client = HydrusNetwork(
|
||||
url=hydrus_url, access_key=args.access_key, timeout=args.timeout
|
||||
url=hydrus_url,
|
||||
access_key=args.access_key,
|
||||
timeout=args.timeout
|
||||
)
|
||||
meta_response = client.fetch_file_metadata(hashes=[file_hash], include_mime=True)
|
||||
entries = meta_response.get("metadata") if isinstance(meta_response, dict) else None
|
||||
meta_response = client.fetch_file_metadata(
|
||||
hashes=[file_hash],
|
||||
include_mime=True
|
||||
)
|
||||
entries = meta_response.get("metadata") if isinstance(
|
||||
meta_response,
|
||||
dict
|
||||
) else None
|
||||
if isinstance(entries, list) and entries:
|
||||
entry = entries[0]
|
||||
ext_value = _normalise_ext(
|
||||
entry.get("ext") if isinstance(entry, dict) else None
|
||||
entry.get("ext") if isinstance(entry,
|
||||
dict) else None
|
||||
)
|
||||
if ext_value:
|
||||
resolved_suffix = ext_value
|
||||
else:
|
||||
mime_value = entry.get("mime") if isinstance(entry, dict) else None
|
||||
mime_value = entry.get("mime"
|
||||
) if isinstance(entry,
|
||||
dict) else None
|
||||
resolved_suffix = _extension_from_mime(mime_value)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
if os.environ.get("DOWNLOW_DEBUG"):
|
||||
@@ -1072,7 +1268,8 @@ def hydrus_export(args, _parser) -> int:
|
||||
source_suffix = resolved_suffix
|
||||
|
||||
suffix = source_suffix or ".hydrus"
|
||||
if suffix and output_path.suffix.lower() in {"", ".bin"}:
|
||||
if suffix and output_path.suffix.lower() in {"",
|
||||
".bin"}:
|
||||
if output_path.suffix.lower() != suffix.lower():
|
||||
output_path = output_path.with_suffix(suffix)
|
||||
target_dir = output_path.parent
|
||||
@@ -1082,7 +1279,11 @@ def hydrus_export(args, _parser) -> int:
|
||||
ensure_directory(temp_dir)
|
||||
except RuntimeError:
|
||||
temp_dir = target_dir
|
||||
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=suffix, dir=str(temp_dir))
|
||||
temp_file = tempfile.NamedTemporaryFile(
|
||||
delete=False,
|
||||
suffix=suffix,
|
||||
dir=str(temp_dir)
|
||||
)
|
||||
temp_path = Path(temp_file.name)
|
||||
temp_file.close()
|
||||
downloaded_bytes = 0
|
||||
@@ -1090,7 +1291,12 @@ def hydrus_export(args, _parser) -> int:
|
||||
"Hydrus-Client-API-Access-Key": args.access_key,
|
||||
}
|
||||
try:
|
||||
downloaded_bytes = download_hydrus_file(args.file_url, headers, temp_path, args.timeout)
|
||||
downloaded_bytes = download_hydrus_file(
|
||||
args.file_url,
|
||||
headers,
|
||||
temp_path,
|
||||
args.timeout
|
||||
)
|
||||
if os.environ.get("DOWNLOW_DEBUG"):
|
||||
log(f"hydrus-export downloaded {downloaded_bytes} bytes", file=sys.stderr)
|
||||
except httpx.RequestError as exc:
|
||||
@@ -1139,20 +1345,24 @@ def hydrus_export(args, _parser) -> int:
|
||||
if completed.returncode != 0:
|
||||
error_details = ffmpeg_log or (completed.stdout or "").strip()
|
||||
raise RuntimeError(
|
||||
f"ffmpeg failed with exit code {completed.returncode}"
|
||||
+ (f": {error_details}" if error_details else "")
|
||||
f"ffmpeg failed with exit code {completed.returncode}" +
|
||||
(f": {error_details}" if error_details else "")
|
||||
)
|
||||
shutil.move(str(converted_tmp), str(final_target))
|
||||
result_path = final_target
|
||||
apply_mutagen_metadata(result_path, ffmpeg_metadata, args.format)
|
||||
result_size = result_path.stat().st_size if result_path.exists() else None
|
||||
payload: dict[str, object] = {"output": str(result_path)}
|
||||
payload: dict[str,
|
||||
object] = {
|
||||
"output": str(result_path)
|
||||
}
|
||||
if downloaded_bytes:
|
||||
payload["source_bytes"] = downloaded_bytes
|
||||
if result_size is not None:
|
||||
payload["size_bytes"] = result_size
|
||||
if metadata_payload:
|
||||
payload["metadata_keys"] = sorted(ffmpeg_metadata.keys()) if ffmpeg_metadata else []
|
||||
payload["metadata_keys"] = sorted(ffmpeg_metadata.keys()
|
||||
) if ffmpeg_metadata else []
|
||||
log(json.dumps(payload, ensure_ascii=False))
|
||||
if ffmpeg_log:
|
||||
log(ffmpeg_log, file=sys.stderr)
|
||||
@@ -1179,7 +1389,6 @@ def hydrus_export(args, _parser) -> int:
|
||||
# This section consolidates functions formerly in hydrus_wrapper.py
|
||||
# Provides: supported filetypes, client initialization, caching, service resolution
|
||||
|
||||
|
||||
# Official Hydrus supported filetypes
|
||||
# Source: https://hydrusnetwork.github.io/hydrus/filetypes.html
|
||||
SUPPORTED_FILETYPES = {
|
||||
@@ -1240,9 +1449,11 @@ SUPPORTED_FILETYPES = {
|
||||
".pdf": "application/pdf",
|
||||
".epub": "application/epub+zip",
|
||||
".djvu": "image/vnd.djvu",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".docx":
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".pptx":
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".doc": "application/msword",
|
||||
".xls": "application/vnd.ms-excel",
|
||||
".ppt": "application/vnd.ms-powerpoint",
|
||||
@@ -1271,9 +1482,9 @@ SUPPORTED_FILETYPES = {
|
||||
# Flatten to get all supported extensions
|
||||
ALL_SUPPORTED_EXTENSIONS = set(GLOBAL_SUPPORTED_EXTENSIONS)
|
||||
|
||||
|
||||
# Global Hydrus client cache to reuse session keys
|
||||
_hydrus_client_cache: dict[str, Any] = {}
|
||||
_hydrus_client_cache: dict[str,
|
||||
Any] = {}
|
||||
|
||||
# Cache Hydrus availability across the session
|
||||
_HYDRUS_AVAILABLE: Optional[bool] = None
|
||||
@@ -1287,7 +1498,10 @@ def reset_cache() -> None:
|
||||
_HYDRUS_UNAVAILABLE_REASON = None
|
||||
|
||||
|
||||
def is_available(config: dict[str, Any], use_cache: bool = True) -> tuple[bool, Optional[str]]:
|
||||
def is_available(config: dict[str,
|
||||
Any],
|
||||
use_cache: bool = True) -> tuple[bool,
|
||||
Optional[str]]:
|
||||
"""Check if Hydrus is available and accessible.
|
||||
|
||||
Performs a lightweight probe to verify:
|
||||
@@ -1310,7 +1524,7 @@ def is_available(config: dict[str, Any], use_cache: bool = True) -> tuple[bool,
|
||||
return _HYDRUS_AVAILABLE, _HYDRUS_UNAVAILABLE_REASON
|
||||
|
||||
# Use new config helpers first, fallback to old method
|
||||
from config import get_hydrus_url, get_hydrus_access_key
|
||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||
|
||||
url = (get_hydrus_url(config, "home") or "").strip()
|
||||
if not url:
|
||||
@@ -1399,7 +1613,7 @@ def get_client(config: dict[str, Any]) -> HydrusNetwork:
|
||||
if not available:
|
||||
raise RuntimeError(f"Hydrus is unavailable: {reason}")
|
||||
|
||||
from config import get_hydrus_url, get_hydrus_access_key
|
||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||
|
||||
# Use new config helpers
|
||||
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
|
||||
@@ -1446,7 +1660,8 @@ def get_tag_service_name(config: dict[str, Any]) -> str:
|
||||
return "my tags"
|
||||
|
||||
|
||||
def get_tag_service_key(client: HydrusNetwork, fallback_name: str = "my tags") -> Optional[str]:
|
||||
def get_tag_service_key(client: HydrusNetwork,
|
||||
fallback_name: str = "my tags") -> Optional[str]:
|
||||
"""Get the service key for a named tag service.
|
||||
|
||||
Queries the Hydrus client's services and finds the service key matching
|
||||
@@ -1498,7 +1713,11 @@ CHUNK_SIZE = 1024 * 1024 # 1 MiB
|
||||
|
||||
|
||||
def download_hydrus_file(
|
||||
file_url: str, headers: dict[str, str], destination: Path, timeout: float
|
||||
file_url: str,
|
||||
headers: dict[str,
|
||||
str],
|
||||
destination: Path,
|
||||
timeout: float
|
||||
) -> int:
|
||||
"""Download *file_url* into *destination* returning the byte count with progress bar."""
|
||||
from SYS.progress import print_progress, print_final_progress
|
||||
|
||||
164
API/alldebrid.py
164
API/alldebrid.py
@@ -30,19 +30,24 @@ _SUPPORTED_HOSTERS_CACHE: Optional[Dict[str, Dict[str, Any]]] = None
|
||||
_CACHE_TIMESTAMP: float = 0
|
||||
_CACHE_DURATION: float = 3600 # 1 hour
|
||||
|
||||
|
||||
# Cache for init-time connectivity checks (api_key fingerprint -> (ok, reason))
|
||||
_INIT_CHECK_CACHE: Dict[str, Tuple[bool, Optional[str]]] = {}
|
||||
_INIT_CHECK_CACHE: Dict[str,
|
||||
Tuple[bool,
|
||||
Optional[str]]] = {}
|
||||
|
||||
|
||||
def _ping_alldebrid(base_url: str) -> Tuple[bool, Optional[str]]:
|
||||
"""Ping the AllDebrid API base URL (no API key required)."""
|
||||
try:
|
||||
url = str(base_url or "").rstrip("/") + "/ping"
|
||||
with HTTPClient(timeout=10.0, headers={"User-Agent": "downlow/1.0"}) as client:
|
||||
with HTTPClient(timeout=10.0,
|
||||
headers={
|
||||
"User-Agent": "downlow/1.0"
|
||||
}) as client:
|
||||
response = client.get(url)
|
||||
data = json.loads(response.content.decode("utf-8"))
|
||||
if data.get("status") == "success" and data.get("data", {}).get("ping") == "pong":
|
||||
if data.get("status") == "success" and data.get("data",
|
||||
{}).get("ping") == "pong":
|
||||
return True, None
|
||||
return False, "Invalid API response"
|
||||
except Exception as exc:
|
||||
@@ -107,10 +112,12 @@ class AllDebridClient:
|
||||
def _request(
|
||||
self,
|
||||
endpoint: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
params: Optional[Dict[str,
|
||||
Any]] = None,
|
||||
*,
|
||||
method: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
) -> Dict[str,
|
||||
Any]:
|
||||
"""Make a request to AllDebrid API.
|
||||
|
||||
Args:
|
||||
@@ -157,12 +164,19 @@ class AllDebridClient:
|
||||
except Exception as req_err:
|
||||
# Log detailed error info
|
||||
logger.error(
|
||||
f"[AllDebrid] Request error to {endpoint}: {req_err}", exc_info=True
|
||||
f"[AllDebrid] Request error to {endpoint}: {req_err}",
|
||||
exc_info=True
|
||||
)
|
||||
if hasattr(req_err, "response") and req_err.response is not None: # type: ignore
|
||||
if hasattr(req_err,
|
||||
"response"
|
||||
) and req_err.response is not None: # type: ignore
|
||||
try:
|
||||
error_body = req_err.response.content.decode("utf-8") # type: ignore
|
||||
logger.error(f"[AllDebrid] Response body: {error_body[:200]}")
|
||||
error_body = req_err.response.content.decode(
|
||||
"utf-8"
|
||||
) # type: ignore
|
||||
logger.error(
|
||||
f"[AllDebrid] Response body: {error_body[:200]}"
|
||||
)
|
||||
except:
|
||||
pass
|
||||
raise
|
||||
@@ -172,7 +186,9 @@ class AllDebridClient:
|
||||
|
||||
# Check for API errors
|
||||
if data.get("status") == "error":
|
||||
error_msg = data.get("error", {}).get("message", "Unknown error")
|
||||
error_msg = data.get("error",
|
||||
{}).get("message",
|
||||
"Unknown error")
|
||||
logger.error(f"[AllDebrid] API error: {error_msg}")
|
||||
raise AllDebridError(f"AllDebrid API error: {error_msg}")
|
||||
|
||||
@@ -200,11 +216,15 @@ class AllDebridClient:
|
||||
raise AllDebridError(f"Invalid URL: {link}")
|
||||
|
||||
try:
|
||||
response = self._request("link/unlock", {"link": link})
|
||||
response = self._request("link/unlock",
|
||||
{
|
||||
"link": link
|
||||
})
|
||||
|
||||
# Check if unlock was successful
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
|
||||
# AllDebrid returns the download info in 'link' field
|
||||
if "link" in data:
|
||||
@@ -251,10 +271,18 @@ class AllDebridClient:
|
||||
|
||||
for category in ("hosts", "streams", "redirectors"):
|
||||
values = domains.get(category)
|
||||
if isinstance(values, list) and any(str(d).lower() == host for d in values):
|
||||
return {"supported": True, "category": category, "domain": host}
|
||||
if isinstance(values,
|
||||
list) and any(str(d).lower() == host for d in values):
|
||||
return {
|
||||
"supported": True,
|
||||
"category": category,
|
||||
"domain": host
|
||||
}
|
||||
|
||||
return {"supported": False, "domain": host}
|
||||
return {
|
||||
"supported": False,
|
||||
"domain": host
|
||||
}
|
||||
except AllDebridError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
@@ -274,7 +302,8 @@ class AllDebridClient:
|
||||
response = self._request("user")
|
||||
|
||||
if response.get("status") == "success":
|
||||
return response.get("data", {})
|
||||
return response.get("data",
|
||||
{})
|
||||
|
||||
return {}
|
||||
except AllDebridError:
|
||||
@@ -296,8 +325,10 @@ class AllDebridClient:
|
||||
response = self._request("hosts/domains")
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
return data if isinstance(data, dict) else {}
|
||||
data = response.get("data",
|
||||
{})
|
||||
return data if isinstance(data,
|
||||
dict) else {}
|
||||
|
||||
return {}
|
||||
except AllDebridError:
|
||||
@@ -331,10 +362,14 @@ class AllDebridClient:
|
||||
try:
|
||||
# API endpoint: POST /v4/magnet/upload
|
||||
# Format: /magnet/upload?apikey=key&magnets[]=magnet:?xt=...
|
||||
response = self._request("magnet/upload", {"magnets[]": magnet_uri})
|
||||
response = self._request("magnet/upload",
|
||||
{
|
||||
"magnets[]": magnet_uri
|
||||
})
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
magnets = data.get("magnets", [])
|
||||
|
||||
if magnets and len(magnets) > 0:
|
||||
@@ -356,7 +391,10 @@ class AllDebridClient:
|
||||
except Exception as exc:
|
||||
raise AllDebridError(f"Failed to submit magnet: {exc}")
|
||||
|
||||
def magnet_status(self, magnet_id: int, include_files: bool = False) -> Dict[str, Any]:
|
||||
def magnet_status(self,
|
||||
magnet_id: int,
|
||||
include_files: bool = False) -> Dict[str,
|
||||
Any]:
|
||||
"""Get status of a magnet currently being processed or stored.
|
||||
|
||||
Status codes:
|
||||
@@ -396,13 +434,18 @@ class AllDebridClient:
|
||||
self.base_url = self.BASE_URL_V41
|
||||
|
||||
try:
|
||||
response = self._request("magnet/status", {"id": str(magnet_id)})
|
||||
response = self._request("magnet/status",
|
||||
{
|
||||
"id": str(magnet_id)
|
||||
})
|
||||
finally:
|
||||
self.base_url = old_base
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
magnets = data.get("magnets", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
magnets = data.get("magnets",
|
||||
{})
|
||||
|
||||
# Handle both list and dict responses
|
||||
if isinstance(magnets, list) and len(magnets) > 0:
|
||||
@@ -439,7 +482,8 @@ class AllDebridClient:
|
||||
if response.get("status") != "success":
|
||||
return []
|
||||
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
magnets = data.get("magnets", [])
|
||||
|
||||
if isinstance(magnets, list):
|
||||
@@ -459,8 +503,12 @@ class AllDebridClient:
|
||||
raise AllDebridError(f"Failed to list magnets: {exc}")
|
||||
|
||||
def magnet_status_live(
|
||||
self, magnet_id: int, session: Optional[int] = None, counter: int = 0
|
||||
) -> Dict[str, Any]:
|
||||
self,
|
||||
magnet_id: int,
|
||||
session: Optional[int] = None,
|
||||
counter: int = 0
|
||||
) -> Dict[str,
|
||||
Any]:
|
||||
"""Get live status of a magnet using delta sync mode.
|
||||
|
||||
The live mode endpoint provides real-time progress by only sending
|
||||
@@ -493,7 +541,10 @@ class AllDebridClient:
|
||||
old_base = self.base_url
|
||||
self.base_url = self.BASE_URL_V41
|
||||
try:
|
||||
payload: Dict[str, Any] = {"id": str(magnet_id)}
|
||||
payload: Dict[str,
|
||||
Any] = {
|
||||
"id": str(magnet_id)
|
||||
}
|
||||
if session is not None:
|
||||
payload["session"] = str(int(session))
|
||||
payload["counter"] = str(int(counter))
|
||||
@@ -502,7 +553,8 @@ class AllDebridClient:
|
||||
self.base_url = old_base
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
magnets = data.get("magnets", [])
|
||||
|
||||
# For specific magnet id, return the first match from the array.
|
||||
@@ -552,7 +604,8 @@ class AllDebridClient:
|
||||
response = self._request("magnet/files", params)
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
magnets = data.get("magnets", [])
|
||||
|
||||
# Convert list to dict keyed by ID (as string) for easier access
|
||||
@@ -603,10 +656,14 @@ class AllDebridClient:
|
||||
if not hash_value or len(hash_value) < 32:
|
||||
return None
|
||||
|
||||
response = self._request("magnet/instant", {"magnet": hash_value})
|
||||
response = self._request("magnet/instant",
|
||||
{
|
||||
"magnet": hash_value
|
||||
})
|
||||
|
||||
if response.get("status") == "success":
|
||||
data = response.get("data", {})
|
||||
data = response.get("data",
|
||||
{})
|
||||
# Returns 'files' array if available, or empty
|
||||
return data.get("files", [])
|
||||
|
||||
@@ -635,7 +692,10 @@ class AllDebridClient:
|
||||
raise AllDebridError(f"Invalid magnet ID: {magnet_id}")
|
||||
|
||||
try:
|
||||
response = self._request("magnet/delete", {"id": str(magnet_id)})
|
||||
response = self._request("magnet/delete",
|
||||
{
|
||||
"id": str(magnet_id)
|
||||
})
|
||||
|
||||
if response.get("status") == "success":
|
||||
return True
|
||||
@@ -664,7 +724,8 @@ def _get_cached_supported_hosters(api_key: str) -> Set[str]:
|
||||
now = time.time()
|
||||
|
||||
# Return cached result if still valid
|
||||
if _SUPPORTED_HOSTERS_CACHE is not None and (now - _CACHE_TIMESTAMP) < _CACHE_DURATION:
|
||||
if _SUPPORTED_HOSTERS_CACHE is not None and (now -
|
||||
_CACHE_TIMESTAMP) < _CACHE_DURATION:
|
||||
return set(_SUPPORTED_HOSTERS_CACHE.keys())
|
||||
|
||||
# Fetch fresh list from API
|
||||
@@ -686,11 +747,15 @@ def _get_cached_supported_hosters(api_key: str) -> Set[str]:
|
||||
all_domains.update(hosters_dict["streams"])
|
||||
|
||||
# Add redirectors
|
||||
if "redirectors" in hosters_dict and isinstance(hosters_dict["redirectors"], list):
|
||||
if "redirectors" in hosters_dict and isinstance(hosters_dict["redirectors"],
|
||||
list):
|
||||
all_domains.update(hosters_dict["redirectors"])
|
||||
|
||||
# Cache as dict for consistency
|
||||
_SUPPORTED_HOSTERS_CACHE = {domain: {} for domain in all_domains}
|
||||
_SUPPORTED_HOSTERS_CACHE = {
|
||||
domain: {}
|
||||
for domain in all_domains
|
||||
}
|
||||
_CACHE_TIMESTAMP = now
|
||||
|
||||
if all_domains:
|
||||
@@ -905,7 +970,8 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any])
|
||||
0 on success, 1 on failure
|
||||
"""
|
||||
|
||||
def _extract_link_from_args_or_result(result_obj: Any, argv: Sequence[str]) -> Optional[str]:
|
||||
def _extract_link_from_args_or_result(result_obj: Any,
|
||||
argv: Sequence[str]) -> Optional[str]:
|
||||
# Prefer an explicit URL in args.
|
||||
for a in argv or []:
|
||||
if isinstance(a, str) and a.startswith(("http://", "https://")):
|
||||
@@ -923,7 +989,9 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any])
|
||||
# Current config format
|
||||
try:
|
||||
provider_cfg = cfg.get("provider") if isinstance(cfg, dict) else None
|
||||
ad_cfg = provider_cfg.get("alldebrid") if isinstance(provider_cfg, dict) else None
|
||||
ad_cfg = provider_cfg.get("alldebrid"
|
||||
) if isinstance(provider_cfg,
|
||||
dict) else None
|
||||
api_key = ad_cfg.get("api_key") if isinstance(ad_cfg, dict) else None
|
||||
if isinstance(api_key, str) and api_key.strip():
|
||||
return api_key.strip()
|
||||
@@ -943,7 +1011,11 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any])
|
||||
|
||||
return None
|
||||
|
||||
def _add_direct_link_to_result(result_obj: Any, direct_link: str, original_link: str) -> None:
|
||||
def _add_direct_link_to_result(
|
||||
result_obj: Any,
|
||||
direct_link: str,
|
||||
original_link: str
|
||||
) -> None:
|
||||
if not isinstance(direct_link, str) or not direct_link.strip():
|
||||
return
|
||||
if isinstance(result_obj, dict):
|
||||
@@ -963,7 +1035,10 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any])
|
||||
api_key = _get_alldebrid_api_key_from_config(config)
|
||||
|
||||
if not api_key:
|
||||
log("AllDebrid API key not configured (provider.alldebrid.api_key)", file=sys.stderr)
|
||||
log(
|
||||
"AllDebrid API key not configured (provider.alldebrid.api_key)",
|
||||
file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
|
||||
# Try to unlock the link
|
||||
@@ -995,7 +1070,12 @@ def _register_unlock_link():
|
||||
from cmdlet import register
|
||||
|
||||
@register(["unlock-link"])
|
||||
def unlock_link_wrapper(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
def unlock_link_wrapper(
|
||||
result: Any,
|
||||
args: Sequence[str],
|
||||
config: Dict[str,
|
||||
Any]
|
||||
) -> int:
|
||||
"""Wrapper to make unlock_link_cmdlet available as cmdlet."""
|
||||
import pipeline as ctx
|
||||
|
||||
|
||||
797
API/folder.py
797
API/folder.py
File diff suppressed because it is too large
Load Diff
23
API/loc.py
23
API/loc.py
@@ -47,8 +47,10 @@ class LOCClient:
|
||||
*,
|
||||
start: int = 1,
|
||||
count: int = 25,
|
||||
extra_params: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
extra_params: Optional[Dict[str,
|
||||
Any]] = None,
|
||||
) -> Dict[str,
|
||||
Any]:
|
||||
"""Search the Chronicling America collection via LoC JSON API.
|
||||
|
||||
Args:
|
||||
@@ -63,14 +65,17 @@ class LOCClient:
|
||||
|
||||
q = str(query or "").strip()
|
||||
if not q:
|
||||
return {"results": []}
|
||||
return {
|
||||
"results": []
|
||||
}
|
||||
|
||||
params: Dict[str, Any] = {
|
||||
"q": q,
|
||||
"fo": "json",
|
||||
"c": int(count) if int(count) > 0 else 25,
|
||||
"sp": int(start) if int(start) > 0 else 1,
|
||||
}
|
||||
params: Dict[str,
|
||||
Any] = {
|
||||
"q": q,
|
||||
"fo": "json",
|
||||
"c": int(count) if int(count) > 0 else 25,
|
||||
"sp": int(start) if int(start) > 0 else 1,
|
||||
}
|
||||
if extra_params:
|
||||
for k, v in extra_params.items():
|
||||
if v is None:
|
||||
|
||||
Reference in New Issue
Block a user