"""Hydrus API helpers and export utilities.""" from __future__ import annotations import base64 import http.client import json import os import re import shutil import subprocess import sys import time from collections import deque from SYS.logger import log from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS as GLOBAL_SUPPORTED_EXTENSIONS import tempfile import logging from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast from urllib.parse import urlsplit, urlencode, quote, urlunsplit, unquote import httpx logger = logging.getLogger(__name__) from SYS.utils import ( decode_cbor, jsonify, ensure_directory, unique_path, ) from .HTTP import HTTPClient class HydrusRequestError(RuntimeError): """Raised when the Hydrus Client API returns an error response.""" def __init__(self, status: int, message: str, payload: Any | None = None) -> None: super().__init__(f"Hydrus request failed ({status}): {message}") self.status = status self.payload = payload class HydrusConnectionError(HydrusRequestError): """Raised when Hydrus service is unavailable (connection refused, timeout, etc.). This is an expected error when Hydrus is not running and should not include a full traceback in logs. """ def __init__(self, message: str) -> None: super().__init__(0, message, None) # status 0 indicates connection error self.is_connection_error = True @dataclass(slots=True) class HydrusRequestSpec: method: str endpoint: str query: dict[str, Any] | None = None data: Any | None = None file_path: Path | None = None content_type: str | None = None accept: str | None = "application/cbor" @dataclass(slots=True) class HydrusNetwork: """Thin wrapper around the Hydrus Client API.""" url: str access_key: str = "" timeout: float = 9.0 instance_name: str = "" # Optional store name (e.g., 'home') for namespaced logs scheme: str = field(init=False) hostname: str = field(init=False) port: int = field(init=False) base_path: str = field(init=False) _session_key: str = field(init=False, default="", repr=False) # Cached session key def __post_init__(self) -> None: if not self.url: raise ValueError("Hydrus base URL is required") self.url = self.url.rstrip("/") parsed = urlsplit(self.url) if parsed.scheme not in {"http", "https"}: raise ValueError("Hydrus base URL must use http or https") self.scheme = parsed.scheme self.hostname = parsed.hostname or "localhost" self.port = parsed.port or (443 if self.scheme == "https" else 80) self.base_path = parsed.path.rstrip("/") self.access_key = self.access_key or "" self.instance_name = str(self.instance_name or "").strip() def _log_prefix(self) -> str: if self.instance_name: return f"[hydrusnetwork:{self.instance_name}]" return f"[hydrusnetwork:{self.hostname}:{self.port}]" # ------------------------------------------------------------------ # low-level helpers # ------------------------------------------------------------------ def _build_path(self, endpoint: str, query: dict[str, Any] | None = None) -> str: path = endpoint if endpoint.startswith("/") else f"/{endpoint}" if self.base_path: path = f"{self.base_path}{path}" if query: encoded = urlencode(query, doseq=True) if encoded: path = f"{path}?{encoded}" return path def _perform_request(self, spec: HydrusRequestSpec) -> Any: headers: dict[str, str] = {} # Use session key if available, otherwise use access key if self._session_key: headers["Hydrus-Client-API-Session-Key"] = self._session_key elif self.access_key: headers["Hydrus-Client-API-Access-Key"] = self.access_key if spec.accept: headers["Accept"] = spec.accept path = self._build_path(spec.endpoint, spec.query) url = f"{self.scheme}://{self.hostname}:{self.port}{path}" # Log request details logger.debug( f"{self._log_prefix()} {spec.method} {spec.endpoint} (auth: {'session_key' if self._session_key else 'access_key' if self.access_key else 'none'})" ) status = 0 reason = "" body = b"" content_type = "" try: with HTTPClient(timeout=self.timeout, headers=headers, verify_ssl=False) as client: response = None if spec.file_path is not None: file_path = Path(spec.file_path) if not file_path.is_file(): error_msg = f"Upload file not found: {file_path}" logger.error(f"{self._log_prefix()} {error_msg}") raise FileNotFoundError(error_msg) file_size = file_path.stat().st_size headers["Content-Type" ] = spec.content_type or "application/octet-stream" # Do not set Content-Length when streaming an iterator body. # If the file size changes between stat() and read() (or the source is truncated), # h11 will raise: "Too little data for declared Content-Length". # Let httpx choose chunked transfer encoding for safety. headers.pop("Content-Length", None) logger.debug( f"{self._log_prefix()} Uploading file {file_path.name} ({file_size} bytes)" ) # Stream upload body with a stderr progress bar (pipeline-safe). from SYS.models import ProgressBar bar = ProgressBar() # Keep the PipelineLiveProgress transfer line clean: show the file name. # (The hydrus instance/service is already visible in the logs above.) label = str(getattr(file_path, "name", None) or "upload") start_t = time.time() last_render_t = [start_t] sent = [0] def _render_progress(final: bool = False) -> None: if file_size <= 0: return now = time.time() if not final and (now - float(last_render_t[0])) < 0.25: return last_render_t[0] = now bar.update( downloaded=int(sent[0]), total=int(file_size), label=str(label), file=sys.stderr, ) if final: bar.finish() def file_gen(): try: with file_path.open("rb") as handle: while True: chunk = handle.read(256 * 1024) if not chunk: break sent[0] += len(chunk) _render_progress(final=False) yield chunk finally: _render_progress(final=True) response = client.request( spec.method, url, content=file_gen(), headers=headers, raise_for_status=False, log_http_errors=False, ) else: content = None json_data = None if spec.data is not None: if isinstance(spec.data, (bytes, bytearray)): content = spec.data else: json_data = spec.data # Hydrus expects JSON bodies to be sent with Content-Type: application/json. # httpx will usually set this automatically, but we set it explicitly to # match the Hydrus API docs and avoid edge cases. headers.setdefault("Content-Type", "application/json") logger.debug( f"{self._log_prefix()} Request body size: {len(content) if content else 'json'}" ) response = client.request( spec.method, url, content=content, json=json_data, headers=headers, raise_for_status=False, log_http_errors=False, ) status = response.status_code reason = response.reason_phrase body = response.content content_type = response.headers.get("Content-Type", "") or "" logger.debug( f"{self._log_prefix()} Response {status} {reason} ({len(body)} bytes)" ) except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) as exc: msg = f"Hydrus unavailable: {exc}" logger.warning(f"{self._log_prefix()} {msg}") raise HydrusConnectionError(msg) from exc except Exception as exc: logger.error(f"{self._log_prefix()} Connection error: {exc}", exc_info=True) raise payload: Any payload = {} if body: content_main = content_type.split(";", 1)[0].strip().lower() if "json" in content_main: try: payload = json.loads(body.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError): payload = body.decode("utf-8", "replace") elif "cbor" in content_main: try: payload = decode_cbor(body) except Exception: payload = body else: payload = body if status >= 400: message = "" if isinstance(payload, dict): message = str(payload.get("message") or payload.get("error") or payload) elif isinstance(payload, str): message = payload else: message = reason or "HTTP error" # Some endpoints are naturally "missing" sometimes and should not spam logs. if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path": return {} logger.error(f"{self._log_prefix()} HTTP {status}: {message}") # Handle expired session key (419) by clearing cache and retrying once if status == 419 and self._session_key and "session" in message.lower(): logger.warning( f"{self._log_prefix()} Session key expired, acquiring new one and retrying..." ) self._session_key = "" # Clear expired session key try: self._acquire_session_key() # Retry the request with new session key return self._perform_request(spec) except Exception as retry_error: logger.error( f"{self._log_prefix()} Retry failed: {retry_error}", exc_info=True ) # If retry fails, raise the original error raise HydrusRequestError(status, message, payload) from retry_error raise HydrusRequestError(status, message, payload) return payload def _acquire_session_key(self) -> str: """Acquire a session key from the Hydrus API using the access key. Session keys are temporary authentication tokens that expire after 24 hours of inactivity, client restart, or if the access key is deleted. They are more secure than passing access keys in every request. Returns the session key string. Raises HydrusRequestError if the request fails. """ if not self.access_key: raise HydrusRequestError( 401, "Cannot acquire session key: no access key configured" ) # Temporarily use access key to get session key original_session_key = self._session_key try: self._session_key = "" # Clear session key to use access key for this request result = self._get("/session_key") session_key = result.get("session_key") if not session_key: raise HydrusRequestError( 500, "Session key response missing 'session_key' field", result ) self._session_key = session_key return session_key except HydrusRequestError: self._session_key = original_session_key raise except Exception as e: self._session_key = original_session_key raise HydrusRequestError(500, f"Failed to acquire session key: {e}") def ensure_session_key(self) -> str: """Ensure a valid session key exists, acquiring one if needed. Returns the session key. If one is already cached, returns it. Otherwise acquires a new session key from the API. """ if self._session_key: return self._session_key return self._acquire_session_key() def _get(self, endpoint: str, *, query: dict[str, Any] | None = None) -> dict[str, Any]: spec = HydrusRequestSpec("GET", endpoint, query=query) return cast(dict[str, Any], self._perform_request(spec)) def _post( self, endpoint: str, *, data: dict[str, Any] | None = None, file_path: Path | None = None, content_type: str | None = None, ) -> dict[str, Any]: spec = HydrusRequestSpec( "POST", endpoint, data=data, file_path=file_path, content_type=content_type ) return cast(dict[str, Any], self._perform_request(spec)) def _ensure_hashes(self, hash: Union[str, Iterable[str]]) -> list[str]: if isinstance(hash, str): return [hash] return list(hash) def _append_access_key(self, url: str) -> str: if not self.access_key: return url separator = "&" if "?" in url else "?" # Use the correct parameter name for Hydrus API compatibility return f"{url}{separator}access_key={quote(self.access_key)}" def add_file(self, path: Union[str, Path]) -> dict[str, Any]: """Add a file to Hydrus using the octet-stream upload mode. This mirrors the Hydrus API POST /add_files/add_file behavior when sending the file bytes as the POST body. The method accepts either a filesystem `Path` or a string path and will raise FileNotFoundError if the target path is not a readable file. """ # Accept both Path and str for convenience file_path = Path(path) if not isinstance(path, Path) else path if not file_path.is_file(): raise FileNotFoundError(f"Upload file not found: {file_path}") # Forward as file_path so the request body is streamed as application/octet-stream return self._post("/add_files/add_file", file_path=file_path) def undelete_files(self, hashes: Union[str, Iterable[str]]) -> dict[str, Any]: """Restore files from Hydrus trash back into 'my files'. Hydrus Client API: POST /add_files/undelete_files Required JSON args: {"hashes": [, ...]} """ hash_list = self._ensure_hashes(hashes) body = { "hashes": hash_list } return self._post("/add_files/undelete_files", data=body) def delete_files( self, hashes: Union[str, Iterable[str]], *, reason: str | None = None ) -> dict[str, Any]: """Delete files in Hydrus. Hydrus Client API: POST /add_files/delete_files Required JSON args: {"hashes": [, ...]} Optional JSON args: {"reason": "..."} """ hash_list = self._ensure_hashes(hashes) body: dict[str, Any] = { "hashes": hash_list } if isinstance(reason, str) and reason.strip(): body["reason"] = reason.strip() return self._post("/add_files/delete_files", data=body) def clear_file_deletion_record(self, hashes: Union[str, Iterable[str]]) -> dict[str, Any]: """Clear Hydrus's file deletion record for the provided hashes. Hydrus Client API: POST /add_files/clear_file_deletion_record Required JSON args: {"hashes": [, ...]} """ hash_list = self._ensure_hashes(hashes) body = { "hashes": hash_list } return self._post("/add_files/clear_file_deletion_record", data=body) def add_tag( self, hash: Union[str, Iterable[str]], tags: Iterable[str], service_name: str ) -> dict[str, Any]: hash = self._ensure_hashes(hash) body = { "hashes": hash, "service_names_to_tags": { service_name: list(tags) } } return self._post("/add_tags/add_tags", data=body) def delete_tag( self, file_hashes: Union[str, Iterable[str]], tags: Iterable[str], service_name: str, *, action: int = 1, ) -> dict[str, Any]: hashes = self._ensure_hashes(file_hashes) body = { "hashes": hashes, "service_names_to_actions_to_tags": { service_name: { action: list(tags) } }, } return self._post("/add_tags/add_tags", data=body) def add_tags_by_key( self, hash: Union[str, Iterable[str]], tags: Iterable[str], service_key: str ) -> dict[str, Any]: hash = self._ensure_hashes(hash) body = { "hashes": hash, "service_keys_to_tags": { service_key: list(tags) } } return self._post("/add_tags/add_tags", data=body) def delete_tags_by_key( self, file_hashes: Union[str, Iterable[str]], tags: Iterable[str], service_key: str, *, action: int = 1, ) -> dict[str, Any]: hashes = self._ensure_hashes(file_hashes) body = { "hashes": hashes, "service_keys_to_actions_to_tags": { service_key: { action: list(tags) } }, } return self._post("/add_tags/add_tags", data=body) def mutate_tags_by_key( self, hash: Union[str, Iterable[str]], service_key: str, *, add_tags: Optional[Iterable[str]] = None, remove_tags: Optional[Iterable[str]] = None, ) -> dict[str, Any]: """Add or remove tags with a single /add_tags/add_tags call. Hydrus Client API: POST /add_tags/add_tags Use `service_keys_to_actions_to_tags` so the client can apply additions and removals in a single request (action '0' = add, '1' = remove). """ hash_list = self._ensure_hashes(hash) def _clean(tags: Optional[Iterable[str]]) -> list[str]: if not tags: return [] clean_list: list[str] = [] for tag in tags: if not isinstance(tag, str): continue text = tag.strip() if not text: continue clean_list.append(text) return clean_list actions: dict[str, list[str]] = {} adds = _clean(add_tags) removes = _clean(remove_tags) if adds: actions["0"] = adds if removes: actions["1"] = removes if not actions: return {} body = { "hashes": hash_list, "service_keys_to_actions_to_tags": { str(service_key): actions }, } return self._post("/add_tags/add_tags", data=body) def associate_url(self, file_hashes: Union[str, Iterable[str]], url: str) -> dict[str, Any]: hashes = self._ensure_hashes(file_hashes) if len(hashes) == 1: body = { "hash": hashes[0], "url_to_add": url } return self._post("/add_urls/associate_url", data=body) results: dict[str, Any] = {} for file_hash in hashes: body = { "hash": file_hash, "url_to_add": url } results[file_hash] = self._post("/add_urls/associate_url", data=body) return { "batched": results } def get_url_info(self, url: str) -> dict[str, Any]: """Get information about a URL. Hydrus Client API: GET /add_urls/get_url_info Docs: https://hydrusnetwork.github.io/hydrus/developer_api.html#add_urls_get_url_info """ url = str(url or "").strip() if not url: raise ValueError("url must not be empty") spec = HydrusRequestSpec( method="GET", endpoint="/add_urls/get_url_info", query={ "url": url }, ) return cast(dict[str, Any], self._perform_request(spec)) def delete_url(self, file_hashes: Union[str, Iterable[str]], url: str) -> dict[str, Any]: hashes = self._ensure_hashes(file_hashes) if len(hashes) == 1: body = { "hash": hashes[0], "url_to_delete": url } return self._post("/add_urls/associate_url", data=body) results: dict[str, Any] = {} for file_hash in hashes: body = { "hash": file_hash, "url_to_delete": url } results[file_hash] = self._post("/add_urls/associate_url", data=body) return { "batched": results } def set_notes( self, file_hash: str, notes: dict[str, str], *, merge_cleverly: bool = False, extend_existing_note_if_possible: bool = True, conflict_resolution: int = 3, ) -> dict[str, Any]: """Add or update notes associated with a file. Hydrus Client API: POST /add_notes/set_notes Required JSON args: {"hash": , "notes": {name: text}} """ if not notes: raise ValueError("notes mapping must not be empty") file_hash = str(file_hash or "").strip().lower() if not file_hash: raise ValueError("file_hash must not be empty") body: dict[str, Any] = { "hash": file_hash, "notes": notes } if merge_cleverly: body["merge_cleverly"] = True body["extend_existing_note_if_possible"] = bool( extend_existing_note_if_possible ) body["conflict_resolution"] = int(conflict_resolution) return self._post("/add_notes/set_notes", data=body) def delete_notes( self, file_hash: str, note_names: Sequence[str], ) -> dict[str, Any]: """Delete notes associated with a file. Hydrus Client API: POST /add_notes/delete_notes Required JSON args: {"hash": , "note_names": [..]} """ names = [str(name) for name in note_names if str(name or "").strip()] if not names: raise ValueError("note_names must not be empty") file_hash = str(file_hash or "").strip().lower() if not file_hash: raise ValueError("file_hash must not be empty") body = { "hash": file_hash, "note_names": names } return self._post("/add_notes/delete_notes", data=body) def get_file_relationships(self, file_hash: str) -> dict[str, Any]: query = { "hash": file_hash } return self._get( "/manage_file_relationships/get_file_relationships", query=query ) def set_relationship( self, hash_a: str, hash_b: str, relationship: Union[str, int], do_default_content_merge: bool = False, ) -> dict[str, Any]: """Set a relationship between two files in Hydrus. This wraps Hydrus Client API: POST /manage_file_relationships/set_file_relationships. Hydrus relationship enum (per Hydrus developer API docs): - 0: set as potential duplicates - 1: set as false positives - 2: set as same quality (duplicates) - 3: set as alternates - 4: set A as better (duplicates) Args: hash_a: First file SHA256 hex hash_b: Second file SHA256 hex relationship: Relationship type as string or integer enum (0-4) do_default_content_merge: Whether to perform default duplicate content merge Returns: Response from Hydrus API """ # Convert string relationship types to integers if isinstance(relationship, str): rel_map = { # Potential duplicates "potential": 0, "potentials": 0, "potential duplicate": 0, "potential duplicates": 0, # False positives "false positive": 1, "false_positive": 1, "false positives": 1, "false_positives": 1, "not related": 1, "not_related": 1, # Duplicates (same quality) "duplicate": 2, "duplicates": 2, "same quality": 2, "same_quality": 2, "equal": 2, # Alternates "alt": 3, "alternate": 3, "alternates": 3, "alternative": 3, "related": 3, # Better/worse (duplicates) "better": 4, "a better": 4, "a_better": 4, # Back-compat: some older call sites used 'king' for primary. # Hydrus does not accept 'king' as a relationship; this maps to 'A is better'. "king": 4, } relationship = rel_map.get( relationship.lower().strip(), 3 ) # Default to alternates body = { "relationships": [ { "hash_a": hash_a, "hash_b": hash_b, "relationship": relationship, "do_default_content_merge": do_default_content_merge, } ] } return self._post( "/manage_file_relationships/set_file_relationships", data=body ) def get_services(self) -> dict[str, Any]: return self._get("/get_services") def search_files( self, tags: Sequence[Any], *, file_service_name: str | None = None, return_hashes: bool = False, return_file_ids: bool = True, return_file_count: bool = False, include_current_tags: bool | None = None, include_pending_tags: bool | None = None, file_sort_type: int | None = None, file_sort_asc: bool | None = None, file_sort_key: str | None = None, ) -> dict[str, Any]: if not tags: raise ValueError("tags must not be empty") query: dict[str, Any] = {} query_fields = [ ("tags", tags, lambda v: json.dumps(list(v))), ("file_service_name", file_service_name, lambda v: v), ("return_hashes", return_hashes, lambda v: "true" if v else None), ("return_file_ids", return_file_ids, lambda v: "true" if v else None), ("return_file_count", return_file_count, lambda v: "true" if v else None), ( "include_current_tags", include_current_tags, lambda v: "true" if v else "false" if v is not None else None, ), ( "include_pending_tags", include_pending_tags, lambda v: "true" if v else "false" if v is not None else None, ), ( "file_sort_type", file_sort_type, lambda v: str(v) if v is not None else None ), ( "file_sort_asc", file_sort_asc, lambda v: "true" if v else "false" if v is not None else None, ), ("file_sort_key", file_sort_key, lambda v: v), ] for key, value, formatter in query_fields: if value is None or value == []: continue formatted = formatter(value) if formatted is not None: query[key] = formatted return self._get("/get_files/search_files", query=query) def fetch_file_metadata( self, *, file_ids: Sequence[int] | None = None, hashes: Sequence[str] | None = None, include_service_keys_to_tags: bool = True, include_file_url: bool = False, include_duration: bool = True, include_size: bool = True, include_mime: bool = False, include_notes: bool = False, ) -> dict[str, Any]: if not file_ids and not hashes: raise ValueError("Either file_ids or hashes must be provided") query: dict[str, Any] = {} query_fields = [ ("file_ids", file_ids, lambda v: json.dumps(list(v))), ("hashes", hashes, lambda v: json.dumps(list(v))), ( "include_service_keys_to_tags", include_service_keys_to_tags, lambda v: "true" if v else None, ), ("include_file_url", include_file_url, lambda v: "true" if v else None), ("include_duration", include_duration, lambda v: "true" if v else None), ("include_size", include_size, lambda v: "true" if v else None), ("include_mime", include_mime, lambda v: "true" if v else None), ("include_notes", include_notes, lambda v: "true" if v else None), ] for key, value, formatter in query_fields: if not value: continue formatted = formatter(value) if formatted is not None: query[key] = formatted return self._get("/get_files/file_metadata", query=query) def get_file_path(self, file_hash: str) -> dict[str, Any]: """Get the local file system path for a given file hash.""" query = { "hash": file_hash } return self._get("/get_files/file_path", query=query) def file_url(self, file_hash: str) -> str: hash_param = quote(file_hash) # Don't append access_key parameter for file downloads - use header instead url = f"{self.url}/get_files/file?hash={hash_param}" return url def thumbnail_url(self, file_hash: str) -> str: hash_param = quote(file_hash) # Don't append access_key parameter for file downloads - use header instead url = f"{self.url}/get_files/thumbnail?hash={hash_param}" return url HydrusCliOptionsT = TypeVar("HydrusCliOptionsT", bound="HydrusCliOptions") @dataclass(slots=True) class HydrusCliOptions: url: str method: str access_key: str accept: str timeout: float content_type: str | None body_bytes: bytes | None = None body_path: Path | None = None debug: bool = False @classmethod def from_namespace( cls: Type[HydrusCliOptionsT], namespace: Any ) -> HydrusCliOptionsT: accept_header = namespace.accept or "application/cbor" body_bytes: bytes | None = None body_path: Path | None = None if namespace.body_file: body_path = Path(namespace.body_file) elif namespace.body is not None: body_bytes = namespace.body.encode("utf-8") return cls( url=namespace.url, method=namespace.method.upper(), access_key=namespace.access_key or "", accept=accept_header, timeout=namespace.timeout, content_type=namespace.content_type, body_bytes=body_bytes, body_path=body_path, debug=bool(os.environ.get("DOWNLOW_DEBUG")), ) def hydrus_request(args, parser) -> int: if args.body and args.body_file: parser.error("Only one of --body or --body-file may be supplied") options = HydrusCliOptions.from_namespace(args) parsed = urlsplit(options.url) if parsed.scheme not in ("http", "https"): parser.error("Only http and https url are supported") if not parsed.hostname: parser.error("Invalid Hydrus URL") headers: dict[str, str] = {} if options.access_key: headers["Hydrus-Client-API-Access-Key"] = options.access_key if options.accept: headers["Accept"] = options.accept request_body_bytes: bytes | None = None body_path: Path | None = None if options.body_path is not None: body_path = options.body_path if not body_path.is_file(): parser.error(f"File not found: {body_path}") headers.setdefault( "Content-Type", options.content_type or "application/octet-stream" ) headers["Content-Length"] = str(body_path.stat().st_size) elif options.body_bytes is not None: request_body_bytes = options.body_bytes headers["Content-Type"] = options.content_type or "application/json" assert request_body_bytes is not None headers["Content-Length"] = str(len(request_body_bytes)) elif options.content_type: headers["Content-Type"] = options.content_type if parsed.username or parsed.password: userinfo = f"{parsed.username or ''}:{parsed.password or ''}".encode("utf-8") headers["Authorization"] = "Basic " + base64.b64encode(userinfo).decode("ascii") path = parsed.path or "/" if parsed.query: path += "?" + parsed.query port = parsed.port if port is None: port = 443 if parsed.scheme == "https" else 80 connection_cls = ( http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection ) host = parsed.hostname or "localhost" connection = connection_cls(host, port, timeout=options.timeout) if options.debug: log( f"Hydrus connecting to {parsed.scheme}://{host}:{port}{path}", file=sys.stderr ) response_bytes: bytes = b"" content_type = "" status = 0 try: if body_path is not None: with body_path.open("rb") as handle: if options.debug: size_hint = headers.get("Content-Length", "unknown") log( f"Hydrus sending file body ({size_hint} bytes)", file=sys.stderr ) connection.putrequest(options.method, path) host_header = host if (parsed.scheme == "http" and port not in (80, None)) or (parsed.scheme == "https" and port not in (443, None)): host_header = f"{host}:{port}" connection.putheader("Host", host_header) for key, value in headers.items(): if value: connection.putheader(key, value) connection.endheaders() while True: chunk = handle.read(65536) if not chunk: break connection.send(chunk) if options.debug: log( "[downlow.py] Hydrus upload complete; awaiting response", file=sys.stderr ) else: if options.debug: size_hint = "none" if request_body_bytes is None else str( len(request_body_bytes) ) log(f"Hydrus sending request body bytes={size_hint}", file=sys.stderr) sanitized_headers = { k: v for k, v in headers.items() if v } connection.request( options.method, path, body=request_body_bytes, headers=sanitized_headers ) response = connection.getresponse() status = response.status response_bytes = response.read() if options.debug: log( f"Hydrus response received ({len(response_bytes)} bytes)", file=sys.stderr ) content_type = response.getheader("Content-Type", "") except (OSError, http.client.HTTPException) as exc: log(f"HTTP error: {exc}", file=sys.stderr) return 1 finally: connection.close() content_type_lower = (content_type or "").split(";", 1)[0].strip().lower() accept_value = options.accept or "" expect_cbor = "cbor" in (content_type_lower or "") or "cbor" in accept_value.lower() payload = None decode_error: Exception | None = None if response_bytes: if expect_cbor: try: payload = decode_cbor(response_bytes) except Exception as exc: # pragma: no cover - library errors surfaced decode_error = exc if payload is None and not expect_cbor: try: payload = json.loads(response_bytes.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError): payload = response_bytes.decode("utf-8", "replace") elif payload is None and expect_cbor and decode_error is not None: log( f"Expected CBOR response but decoding failed: {decode_error}", file=sys.stderr ) return 1 json_ready = jsonify(payload) if isinstance(payload, (dict, list)) else payload if options.debug: log(f"Hydrus {options.method} {options.url} -> {status}", file=sys.stderr) if isinstance(json_ready, (dict, list)): log(json.dumps(json_ready, ensure_ascii=False)) elif json_ready is None: log("{}") else: log(json.dumps({ "value": json_ready }, ensure_ascii=False)) return 0 if 200 <= status < 400 else 1 def hydrus_export(args, _parser) -> int: from SYS.metadata import apply_mutagen_metadata, build_ffmpeg_command, prepare_ffmpeg_metadata output_path: Path = args.output original_suffix = output_path.suffix target_dir = output_path.parent metadata_payload: Optional[dict[str, Any]] = None metadata_raw = getattr(args, "metadata_json", None) if metadata_raw: try: parsed = json.loads(metadata_raw) except json.JSONDecodeError as exc: log(f"Invalid metadata JSON: {exc}", file=sys.stderr) return 1 if isinstance(parsed, dict): metadata_payload = parsed else: log("[downlow.py] Metadata JSON must decode to an object", file=sys.stderr) return 1 ffmpeg_metadata = prepare_ffmpeg_metadata(metadata_payload) def _normalise_ext(value: Optional[str]) -> Optional[str]: if not value: return None cleaned = value.strip() if not cleaned: return None if not cleaned.startswith("."): # tolerate inputs like "mp4" cleaned = "." + cleaned.lstrip(".") return cleaned def _extension_from_mime(mime: Optional[str]) -> Optional[str]: if not mime: return None mime_map = { # Images / bitmaps "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", "image/avif": ".avif", "image/jxl": ".jxl", # JPEG XL "image/bmp": ".bmp", "image/heic": ".heic", "image/heif": ".heif", "image/x-icon": ".ico", "image/vnd.microsoft.icon": ".ico", "image/qoi": ".qoi", # Quite OK Image "image/tiff": ".tiff", "image/svg+xml": ".svg", "image/vnd.adobe.photoshop": ".psd", # Animation / sequence variants "image/apng": ".apng", "image/avif-sequence": ".avifs", "image/heic-sequence": ".heics", "image/heif-sequence": ".heifs", # Video "video/mp4": ".mp4", "video/webm": ".webm", "video/quicktime": ".mov", "video/ogg": ".ogv", "video/mpeg": ".mpeg", "video/x-msvideo": ".avi", "video/x-flv": ".flv", "video/x-matroska": ".mkv", "video/x-ms-wmv": ".wmv", "video/vnd.rn-realvideo": ".rv", # Audio "audio/mpeg": ".mp3", "audio/mp4": ".m4a", "audio/ogg": ".ogg", "audio/flac": ".flac", "audio/wav": ".wav", "audio/x-wav": ".wav", "audio/x-ms-wma": ".wma", "audio/x-tta": ".tta", "audio/vnd.wave": ".wav", "audio/x-wavpack": ".wv", # Documents / office "application/pdf": ".pdf", "application/epub+zip": ".epub", "application/vnd.djvu": ".djvu", "application/rtf": ".rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx", "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx", "application/msword": ".doc", "application/vnd.ms-excel": ".xls", "application/vnd.ms-powerpoint": ".ppt", # Archive / comicbook / zip-like "application/zip": ".zip", "application/x-7z-compressed": ".7z", "application/x-rar-compressed": ".rar", "application/gzip": ".gz", "application/x-tar": ".tar", "application/x-cbz": ".cbz", # often just ZIP with images; CBZ is not an official mime type but used as mapping # App / project / other "application/clip": ".clip", # Clip Studio "application/x-krita": ".kra", "application/x-procreate": ".procreate", "application/x-shockwave-flash": ".swf", } return mime_map.get(mime.lower()) def _extract_hash(file_url: str) -> Optional[str]: match = re.search(r"[?&]hash=([0-9a-fA-F]+)", file_url) return match.group(1) if match else None # Ensure output and temp directories exist using global helper for dir_path in [target_dir, Path(args.tmp_dir) if args.tmp_dir else target_dir]: try: ensure_directory(dir_path) except RuntimeError as exc: log(f"{exc}", file=sys.stderr) return 1 source_suffix = _normalise_ext(getattr(args, "source_ext", None)) if source_suffix and source_suffix.lower() == ".bin": source_suffix = None if source_suffix is None: hydrus_url = getattr(args, "hydrus_url", None) if not hydrus_url: try: from SYS.config import load_config, get_hydrus_url hydrus_url = get_hydrus_url(load_config()) except Exception as exc: hydrus_url = None if os.environ.get("DOWNLOW_DEBUG"): log( f"hydrus-export could not load Hydrus URL: {exc}", file=sys.stderr ) if hydrus_url: try: setattr(args, "hydrus_url", hydrus_url) except Exception: pass resolved_suffix: Optional[str] = None file_hash = getattr(args, "file_hash", None) or _extract_hash(args.file_url) if hydrus_url and file_hash: try: client = HydrusNetwork( url=hydrus_url, access_key=args.access_key, timeout=args.timeout ) meta_response = client.fetch_file_metadata( hashes=[file_hash], include_mime=True ) entries = meta_response.get("metadata") if isinstance( meta_response, dict ) else None if isinstance(entries, list) and entries: entry = entries[0] ext_value = _normalise_ext( entry.get("ext") if isinstance(entry, dict) else None ) if ext_value: resolved_suffix = ext_value else: mime_value = entry.get("mime" ) if isinstance(entry, dict) else None resolved_suffix = _extension_from_mime(mime_value) except Exception as exc: # pragma: no cover - defensive if os.environ.get("DOWNLOW_DEBUG"): log(f"hydrus metadata fetch failed: {exc}", file=sys.stderr) if not resolved_suffix: fallback_suffix = _normalise_ext(original_suffix) if fallback_suffix and fallback_suffix.lower() == ".bin": fallback_suffix = None resolved_suffix = fallback_suffix or ".hydrus" source_suffix = resolved_suffix suffix = source_suffix or ".hydrus" if suffix and output_path.suffix.lower() in {"", ".bin"}: if output_path.suffix.lower() != suffix.lower(): output_path = output_path.with_suffix(suffix) target_dir = output_path.parent # Determine temp directory (prefer provided tmp_dir, fallback to output location) temp_dir = Path(getattr(args, "tmp_dir", None) or target_dir) try: ensure_directory(temp_dir) except RuntimeError: temp_dir = target_dir temp_file = tempfile.NamedTemporaryFile( delete=False, suffix=suffix, dir=str(temp_dir) ) temp_path = Path(temp_file.name) temp_file.close() downloaded_bytes = 0 headers = { "Hydrus-Client-API-Access-Key": args.access_key, } try: downloaded_bytes = download_hydrus_file( args.file_url, headers, temp_path, args.timeout ) if os.environ.get("DOWNLOW_DEBUG"): log(f"hydrus-export downloaded {downloaded_bytes} bytes", file=sys.stderr) except httpx.RequestError as exc: if temp_path.exists(): temp_path.unlink() log(f"hydrus-export download failed: {exc}", file=sys.stderr) return 1 except Exception as exc: # pragma: no cover - unexpected if temp_path.exists(): temp_path.unlink() log(f"hydrus-export error: {exc}", file=sys.stderr) return 1 ffmpeg_log: Optional[str] = None converted_tmp: Optional[Path] = None try: final_target = unique_path(output_path) if args.format == "copy": shutil.move(str(temp_path), str(final_target)) result_path = final_target else: ffmpeg_path = shutil.which("ffmpeg") if not ffmpeg_path: raise RuntimeError("ffmpeg executable not found in PATH") converted_tmp = final_target.with_suffix(final_target.suffix + ".part") if converted_tmp.exists(): converted_tmp.unlink() max_width = args.max_width if args.max_width and args.max_width > 0 else 0 cmd = build_ffmpeg_command( ffmpeg_path, temp_path, converted_tmp, args.format, max_width, metadata=ffmpeg_metadata if ffmpeg_metadata else None, ) if os.environ.get("DOWNLOW_DEBUG"): log(f"ffmpeg command: {' '.join(cmd)}", file=sys.stderr) completed = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, text=True, ) ffmpeg_log = (completed.stderr or "").strip() if completed.returncode != 0: error_details = ffmpeg_log or (completed.stdout or "").strip() raise RuntimeError( f"ffmpeg failed with exit code {completed.returncode}" + (f": {error_details}" if error_details else "") ) shutil.move(str(converted_tmp), str(final_target)) result_path = final_target apply_mutagen_metadata(result_path, ffmpeg_metadata, args.format) result_size = result_path.stat().st_size if result_path.exists() else None payload: dict[str, object] = { "output": str(result_path) } if downloaded_bytes: payload["source_bytes"] = downloaded_bytes if result_size is not None: payload["size_bytes"] = result_size if metadata_payload: payload["metadata_keys"] = sorted(ffmpeg_metadata.keys() ) if ffmpeg_metadata else [] log(json.dumps(payload, ensure_ascii=False)) if ffmpeg_log: log(ffmpeg_log, file=sys.stderr) return 0 except Exception as exc: log(f"hydrus-export failed: {exc}", file=sys.stderr) return 1 finally: if temp_path.exists(): try: temp_path.unlink() except OSError: pass if converted_tmp and converted_tmp.exists(): try: converted_tmp.unlink() except OSError: pass # ============================================================================ # Hydrus Wrapper Functions - Utilities for client initialization and config # ============================================================================ # This section consolidates functions formerly in hydrus_wrapper.py # Provides: supported filetypes, client initialization, caching, service resolution # Official Hydrus supported filetypes # Source: https://hydrusnetwork.github.io/hydrus/filetypes.html SUPPORTED_FILETYPES = { # Images "image": { ".jpeg": "image/jpeg", ".jpg": "image/jpeg", ".png": "image/png", ".gif": "image/gif", ".webp": "image/webp", ".avif": "image/avif", ".jxl": "image/jxl", ".bmp": "image/bmp", ".heic": "image/heic", ".heif": "image/heif", ".ico": "image/x-icon", ".qoi": "image/qoi", ".tiff": "image/tiff", }, # Animated Images "animation": { ".apng": "image/apng", ".avifs": "image/avif-sequence", ".heics": "image/heic-sequence", ".heifs": "image/heif-sequence", }, # Video "video": { ".mp4": "video/mp4", ".webm": "video/webm", ".mkv": "video/x-matroska", ".avi": "video/x-msvideo", ".flv": "video/x-flv", ".mov": "video/quicktime", ".mpeg": "video/mpeg", ".ogv": "video/ogg", ".rm": "video/vnd.rn-realvideo", ".wmv": "video/x-ms-wmv", }, # Audio "audio": { ".mp3": "audio/mp3", ".ogg": "audio/ogg", ".flac": "audio/flac", ".m4a": "audio/mp4", ".mka": "audio/x-matroska", ".mkv": "audio/x-matroska", ".mp4": "audio/mp4", ".ra": "audio/vnd.rn-realaudio", ".tta": "audio/x-tta", ".wav": "audio/x-wav", ".wv": "audio/wavpack", ".wma": "audio/x-ms-wma", }, # Applications & Documents "application": { ".swf": "application/x-shockwave-flash", ".pdf": "application/pdf", ".epub": "application/epub+zip", ".djvu": "image/vnd.djvu", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", ".doc": "application/msword", ".xls": "application/vnd.ms-excel", ".ppt": "application/vnd.ms-powerpoint", ".rtf": "application/rtf", }, # Image Project Files "project": { ".clip": "application/clip1", ".kra": "application/x-krita", ".procreate": "application/x-procreate1", ".psd": "image/vnd.adobe.photoshop", ".sai2": "application/sai21", ".svg": "image/svg+xml", ".xcf": "application/x-xcf", }, # Archives "archive": { ".cbz": "application/vnd.comicbook+zip", ".7z": "application/x-7z-compressed", ".gz": "application/gzip", ".rar": "application/vnd.rar", ".zip": "application/zip", }, } # Flatten to get all supported extensions ALL_SUPPORTED_EXTENSIONS = set(GLOBAL_SUPPORTED_EXTENSIONS) # Global Hydrus client cache to reuse session keys _hydrus_client_cache: dict[str, Any] = {} # Cache Hydrus availability across the session _HYDRUS_AVAILABLE: Optional[bool] = None _HYDRUS_UNAVAILABLE_REASON: Optional[str] = None def reset_cache() -> None: """Reset the availability cache (useful for testing).""" global _HYDRUS_AVAILABLE, _HYDRUS_UNAVAILABLE_REASON _HYDRUS_AVAILABLE = None _HYDRUS_UNAVAILABLE_REASON = None def is_available(config: dict[str, Any], use_cache: bool = True) -> tuple[bool, Optional[str]]: """Check if Hydrus is available and accessible. Performs a lightweight probe to verify: - Hydrus URL is configured - Can connect to Hydrus URL/port Results are cached per session unless use_cache=False. Args: config: Configuration dict with Hydrus settings use_cache: If True, use cached result from previous probe Returns: Tuple of (is_available: bool, reason: Optional[str]) reason is None if available, or an error message if not """ global _HYDRUS_AVAILABLE, _HYDRUS_UNAVAILABLE_REASON if use_cache and _HYDRUS_AVAILABLE is not None: return _HYDRUS_AVAILABLE, _HYDRUS_UNAVAILABLE_REASON # Use new config helpers first, fallback to old method from SYS.config import get_hydrus_url, get_hydrus_access_key url = (get_hydrus_url(config, "home") or "").strip() if not url: reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)" _HYDRUS_AVAILABLE = False _HYDRUS_UNAVAILABLE_REASON = reason return False, reason access_key = get_hydrus_access_key(config, "home") or "" if not access_key: reason = "Hydrus access key not configured" _HYDRUS_AVAILABLE = False _HYDRUS_UNAVAILABLE_REASON = reason return False, reason timeout_raw = config.get("HydrusNetwork_Request_Timeout") try: timeout = float(timeout_raw) if timeout_raw is not None else 5.0 except (TypeError, ValueError): timeout = 5.0 try: # Simple TCP connection test to URL/port import socket from urllib.parse import urlparse parsed = urlparse(url) hostname = parsed.hostname or "localhost" port = parsed.port or (443 if parsed.scheme == "https" else 80) # Try to connect to the host/port sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: result = sock.connect_ex((hostname, port)) if result == 0: _HYDRUS_AVAILABLE = True _HYDRUS_UNAVAILABLE_REASON = None return True, None else: reason = f"Cannot connect to {hostname}:{port}" _HYDRUS_AVAILABLE = False _HYDRUS_UNAVAILABLE_REASON = reason return False, reason finally: sock.close() except Exception as exc: reason = str(exc) _HYDRUS_AVAILABLE = False _HYDRUS_UNAVAILABLE_REASON = reason return False, reason def is_hydrus_available(config: dict[str, Any]) -> bool: """Check if Hydrus is available without raising. Args: config: Configuration dict Returns: True if Hydrus is available, False otherwise """ available, _ = is_available(config) return available def get_client(config: dict[str, Any]) -> HydrusNetwork: """Create and return a Hydrus client. Uses access-key authentication by default (no session key acquisition). A session key may still be acquired explicitly by calling `HydrusNetwork.ensure_session_key()`. Args: config: Configuration dict with Hydrus settings Returns: HydrusClient instance Raises: RuntimeError: If Hydrus is not configured or unavailable """ # Check availability first - if unavailable, raise immediately available, reason = is_available(config) if not available: raise RuntimeError(f"Hydrus is unavailable: {reason}") from SYS.config import get_hydrus_url, get_hydrus_access_key # Use new config helpers hydrus_url = (get_hydrus_url(config, "home") or "").strip() if not hydrus_url: raise RuntimeError( "Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)" ) access_key = get_hydrus_access_key(config, "home") or "" timeout_raw = config.get("HydrusNetwork_Request_Timeout") try: timeout = float(timeout_raw) if timeout_raw is not None else 60.0 except (TypeError, ValueError): timeout = 60.0 # Create cache key from URL and access key cache_key = f"{hydrus_url}#{access_key}" # Check if we have a cached client if cache_key in _hydrus_client_cache: return _hydrus_client_cache[cache_key] # Create new client client = HydrusNetwork(hydrus_url, access_key, timeout) # Cache the client _hydrus_client_cache[cache_key] = client return client def get_tag_service_name(config: dict[str, Any]) -> str: """Get the name of the tag service to use for tagging operations. Currently always returns "my tags" to avoid remote service errors. Args: config: Configuration dict (not currently used) Returns: Service name string, typically "my tags" """ # Always use 'my tags' to avoid remote service errors return "my tags" def get_tag_service_key(client: HydrusNetwork, fallback_name: str = "my tags") -> Optional[str]: """Get the service key for a named tag service. Queries the Hydrus client's services and finds the service key matching the given name. Args: client: HydrusClient instance fallback_name: Name of the service to find (e.g., "my tags") Returns: Service key string if found, None otherwise """ try: services = client.get_services() except Exception: return None if not isinstance(services, dict): return None # Hydrus returns services grouped by type; walk all lists and match on name for group in services.values(): if not isinstance(group, list): continue for item in group: if not isinstance(item, dict): continue name = str(item.get("name") or "").strip().lower() key = item.get("service_key") or item.get("key") if name == fallback_name.lower() and key: return str(key) return None def is_request_error(exc: Exception) -> bool: """Check if an exception is a Hydrus request error. Args: exc: Exception to check Returns: True if this is a HydrusRequestError """ return isinstance(exc, HydrusRequestError) CHUNK_SIZE = 1024 * 1024 # 1 MiB def download_hydrus_file( file_url: str, headers: dict[str, str], destination: Path, timeout: float ) -> int: """Download *file_url* into *destination* returning the byte count with progress bar.""" from SYS.progress import print_progress, print_final_progress downloaded = 0 start_time = time.time() last_update = start_time # Try to get file size from headers if available file_size = None with HTTPClient(timeout=timeout, headers=headers) as client: response = client.get(file_url) response.raise_for_status() # Try to get size from content-length header try: file_size = int(response.headers.get("content-length", 0)) except (ValueError, TypeError): file_size = None filename = destination.name with destination.open("wb") as handle: for chunk in response.iter_bytes(CHUNK_SIZE): if not chunk: break handle.write(chunk) downloaded += len(chunk) # Update progress every 0.5 seconds if we know total size if file_size: now = time.time() if now - last_update >= 0.5: elapsed = now - start_time speed = downloaded / elapsed if elapsed > 0 else 0 print_progress(filename, downloaded, file_size, speed) last_update = now # Print final progress line if we tracked it if file_size: elapsed = time.time() - start_time print_final_progress(filename, file_size, elapsed) return downloaded # ============================================================================ # Hydrus metadata helpers (moved from SYS.metadata) # ============================================================================ def _normalize_hash(value: Any) -> str: candidate = str(value or "").strip().lower() if not candidate: raise ValueError("Hydrus hash is required") if len(candidate) != 64 or any(ch not in "0123456789abcdef" for ch in candidate): raise ValueError("Hydrus hash must be a 64-character hex string") return candidate def _normalize_tag(tag: Any) -> Optional[str]: if tag is None: return None if isinstance(tag, str): candidate = tag.strip() else: candidate = str(tag).strip() return candidate or None def _dedup_tags_by_namespace(tags: List[str], keep_first: bool = True) -> List[str]: if not tags: return [] namespace_to_tags: Dict[Optional[str], List[Tuple[int, str]]] = {} first_appearance: Dict[Optional[str], int] = {} for idx, tag in enumerate(tags): namespace: Optional[str] = tag.split(":", 1)[0] if ":" in tag else None if namespace not in first_appearance: first_appearance[namespace] = idx if namespace not in namespace_to_tags: namespace_to_tags[namespace] = [] namespace_to_tags[namespace].append((idx, tag)) result: List[Tuple[int, str]] = [] for namespace, tag_list in namespace_to_tags.items(): chosen_tag = tag_list[0][1] if keep_first else tag_list[-1][1] result.append((first_appearance[namespace], chosen_tag)) result.sort(key=lambda x: x[0]) return [tag for _, tag in result] def _extract_tag_services(entry: Dict[str, Any]) -> List[Dict[str, Any]]: tags_section = entry.get("tags") services: List[Dict[str, Any]] = [] if not isinstance(tags_section, dict): return services names_map = tags_section.get("service_keys_to_names") if not isinstance(names_map, dict): names_map = {} def get_record(service_key: Optional[str], service_name: Optional[str]) -> Dict[str, Any]: key_lower = service_key.lower() if isinstance(service_key, str) else None name_lower = service_name.lower() if isinstance(service_name, str) else None for record in services: existing_key = record.get("service_key") if key_lower and isinstance(existing_key, str) and existing_key.lower() == key_lower: if service_name and not record.get("service_name"): record["service_name"] = service_name return record existing_name = record.get("service_name") if name_lower and isinstance(existing_name, str) and existing_name.lower() == name_lower: if service_key and not record.get("service_key"): record["service_key"] = service_key return record record = { "service_key": service_key, "service_name": service_name, "tags": [], } services.append(record) return record def _iter_current_status_lists(container: Any) -> Iterable[List[Any]]: if isinstance(container, dict): for status_key, tags_list in container.items(): if str(status_key) != "0": continue if isinstance(tags_list, list): yield tags_list elif isinstance(container, list): yield container statuses_map = tags_section.get("service_keys_to_statuses_to_tags") if isinstance(statuses_map, dict): for service_key, status_map in statuses_map.items(): record = get_record(service_key if isinstance(service_key, str) else None, names_map.get(service_key)) for tags_list in _iter_current_status_lists(status_map): for tag in tags_list: normalized = _normalize_tag(tag) if normalized: record["tags"].append(normalized) ignored_keys = { "service_keys_to_statuses_to_tags", "service_keys_to_statuses_to_display_tags", "service_keys_to_display_friendly_tags", "service_keys_to_names", "tag_display_types_to_namespaces", "namespace_display_string_lookup", "tag_display_decoration_colour_lookup", } for key, service in tags_section.items(): if key in ignored_keys: continue if isinstance(service, dict): service_key = service.get("service_key") or (key if isinstance(key, str) else None) service_name = service.get("service_name") or service.get("name") or names_map.get(service_key) record = get_record(service_key if isinstance(service_key, str) else None, service_name) storage = service.get("storage_tags") or service.get("statuses_to_tags") or service.get("tags") if isinstance(storage, dict): for tags_list in _iter_current_status_lists(storage): for tag in tags_list: normalized = _normalize_tag(tag) if normalized: record["tags"].append(normalized) elif isinstance(storage, list): for tag in storage: normalized = _normalize_tag(tag) if normalized: record["tags"].append(normalized) for record in services: record["tags"] = _dedup_tags_by_namespace(record["tags"], keep_first=True) return services def _select_primary_tags( services: List[Dict[str, Any]], aggregated: List[str], prefer_service: Optional[str] ) -> Tuple[Optional[str], List[str]]: prefer_lower = prefer_service.lower() if isinstance(prefer_service, str) else None if prefer_lower: for record in services: name = record.get("service_name") if isinstance(name, str) and name.lower() == prefer_lower and record["tags"]: return record.get("service_key"), record["tags"] for record in services: if record["tags"]: return record.get("service_key"), record["tags"] return None, aggregated def _derive_title( tags_primary: List[str], tags_aggregated: List[str], entry: Dict[str, Any] ) -> Optional[str]: for source in (tags_primary, tags_aggregated): for tag in source: namespace, sep, value = tag.partition(":") if sep and namespace and namespace.lower() == "title": cleaned = value.strip() if cleaned: return cleaned for key in ( "title", "display_name", "pretty_name", "original_display_filename", "original_filename", ): value = entry.get(key) if isinstance(value, str): cleaned = value.strip() if cleaned: return cleaned return None def _derive_clip_time( tags_primary: List[str], tags_aggregated: List[str], entry: Dict[str, Any] ) -> Optional[str]: namespaces = {"clip", "clip_time", "cliptime"} for source in (tags_primary, tags_aggregated): for tag in source: namespace, sep, value = tag.partition(":") if sep and namespace and namespace.lower() in namespaces: cleaned = value.strip() if cleaned: return cleaned clip_value = entry.get("clip_time") if isinstance(clip_value, str): cleaned_clip = clip_value.strip() if cleaned_clip: return cleaned_clip return None def _summarize_hydrus_entry( entry: Dict[str, Any], prefer_service: Optional[str] ) -> Tuple[Dict[str, Any], List[str], Optional[str], Optional[str], Optional[str]]: services = _extract_tag_services(entry) aggregated: List[str] = [] seen: Set[str] = set() for record in services: for tag in record["tags"]: if tag not in seen: seen.add(tag) aggregated.append(tag) service_key, primary_tags = _select_primary_tags(services, aggregated, prefer_service) title = _derive_title(primary_tags, aggregated, entry) clip_time = _derive_clip_time(primary_tags, aggregated, entry) summary = dict(entry) if title and not summary.get("title"): summary["title"] = title if clip_time and not summary.get("clip_time"): summary["clip_time"] = clip_time summary["tag_service_key"] = service_key summary["has_current_file_service"] = _has_current_file_service(entry) if "is_local" not in summary: summary["is_local"] = bool(entry.get("is_local")) return summary, primary_tags, service_key, title, clip_time def _looks_like_hash(value: Any) -> bool: if not isinstance(value, str): return False candidate = value.strip().lower() return len(candidate) == 64 and all(ch in "0123456789abcdef" for ch in candidate) def _collect_relationship_hashes(payload: Any, accumulator: Set[str]) -> None: if isinstance(payload, dict): for value in payload.values(): _collect_relationship_hashes(value, accumulator) elif isinstance(payload, (list, tuple, set)): for value in payload: _collect_relationship_hashes(value, accumulator) elif isinstance(payload, str) and _looks_like_hash(payload): accumulator.add(payload) def _generate_hydrus_url_variants(url: str) -> List[str]: seen: Set[str] = set() variants: List[str] = [] def push(candidate: Optional[str]) -> None: if not candidate: return text = candidate.strip() if not text or text in seen: return seen.add(text) variants.append(text) push(url) try: parsed = urlsplit(url) except Exception: return variants if parsed.scheme in {"http", "https"}: alternate_scheme = "https" if parsed.scheme == "http" else "http" push(urlunsplit((alternate_scheme, parsed.netloc, parsed.path, parsed.query, parsed.fragment))) normalised_netloc = parsed.netloc.lower() if normalised_netloc and normalised_netloc != parsed.netloc: push(urlunsplit((parsed.scheme, normalised_netloc, parsed.path, parsed.query, parsed.fragment))) if parsed.path: trimmed_path = parsed.path.rstrip("/") if trimmed_path != parsed.path: push(urlunsplit((parsed.scheme, parsed.netloc, trimmed_path, parsed.query, parsed.fragment))) else: push(urlunsplit((parsed.scheme, parsed.netloc, parsed.path + "/", parsed.query, parsed.fragment))) unquoted_path = unquote(parsed.path) if unquoted_path != parsed.path: push(urlunsplit((parsed.scheme, parsed.netloc, unquoted_path, parsed.query, parsed.fragment))) if parsed.query or parsed.fragment: push(urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))) if parsed.path: unquoted_path = unquote(parsed.path) push(urlunsplit((parsed.scheme, parsed.netloc, unquoted_path, "", ""))) return variants def _build_hydrus_query( hashes: Optional[Sequence[str]], file_ids: Optional[Sequence[int]], include_relationships: bool, minimal: bool, ) -> Dict[str, str]: query: Dict[str, str] = {} if hashes: query["hashes"] = json.dumps([_normalize_hash(h) for h in hashes]) if file_ids: query["file_ids"] = json.dumps([int(fid) for fid in file_ids]) if not query: raise ValueError("hashes or file_ids must be provided") query["include_service_keys_to_tags"] = json.dumps(True) query["include_tag_services"] = json.dumps(True) query["include_file_services"] = json.dumps(True) if include_relationships: query["include_file_relationships"] = json.dumps(True) if not minimal: extras = ( "include_url", "include_size", "include_width", "include_height", "include_duration", "include_mime", "include_has_audio", "include_is_trashed", ) for key in extras: query[key] = json.dumps(True) return query def _fetch_hydrus_entries( client: "HydrusNetwork", hashes: Optional[Sequence[str]], file_ids: Optional[Sequence[int]], include_relationships: bool, minimal: bool, ) -> List[Dict[str, Any]]: if not hashes and not file_ids: return [] spec = HydrusRequestSpec( method="GET", endpoint="/get_files/file_metadata", query=_build_hydrus_query(hashes, file_ids, include_relationships, minimal), ) response = client._perform_request(spec) metadata = response.get("metadata") if isinstance(response, dict) else None if isinstance(metadata, list): return [entry for entry in metadata if isinstance(entry, dict)] return [] def _has_current_file_service(entry: Dict[str, Any]) -> bool: services = entry.get("file_services") if not isinstance(services, dict): return False current = services.get("current") if isinstance(current, dict): for value in current.values(): if value: return True return False if isinstance(current, list): return len(current) > 0 return False def _compute_file_flags(entry: Dict[str, Any]) -> Tuple[bool, bool, bool]: mime = entry.get("mime") mime_lower = mime.lower() if isinstance(mime, str) else "" is_video = mime_lower.startswith("video/") is_audio = mime_lower.startswith("audio/") is_deleted = bool(entry.get("is_trashed")) file_services = entry.get("file_services") if not is_deleted and isinstance(file_services, dict): deleted = file_services.get("deleted") if isinstance(deleted, dict) and deleted: is_deleted = True return is_video, is_audio, is_deleted def fetch_hydrus_metadata(payload: Dict[str, Any]) -> Dict[str, Any]: hash_hex = None raw_hash_value = payload.get("hash") if raw_hash_value is not None: hash_hex = _normalize_hash(raw_hash_value) file_ids: List[int] = [] raw_file_ids = payload.get("file_ids") if isinstance(raw_file_ids, (list, tuple, set)): for value in raw_file_ids: try: file_ids.append(int(value)) except (TypeError, ValueError): continue elif raw_file_ids is not None: try: file_ids.append(int(raw_file_ids)) except (TypeError, ValueError): file_ids = [] raw_file_id = payload.get("file_id") if raw_file_id is not None: try: coerced = int(raw_file_id) except (TypeError, ValueError): coerced = None if coerced is not None and coerced not in file_ids: file_ids.append(coerced) base_url = str(payload.get("api_url") or "").strip() if not base_url: raise ValueError("Hydrus api_url is required") access_key = str(payload.get("access_key") or "").strip() options_raw = payload.get("options") options = options_raw if isinstance(options_raw, dict) else {} prefer_service = options.get("prefer_service_name") if isinstance(prefer_service, str): prefer_service = prefer_service.strip() else: prefer_service = None include_relationships = bool(options.get("include_relationships")) minimal = bool(options.get("minimal")) timeout = float(options.get("timeout") or 60.0) client = HydrusNetwork(base_url, access_key, timeout) hashes: Optional[List[str]] = None if hash_hex: hashes = [hash_hex] if not hashes and not file_ids: raise ValueError("Hydrus hash or file id is required") try: entries = _fetch_hydrus_entries( client, hashes, file_ids or None, include_relationships, minimal ) except HydrusRequestError as exc: raise RuntimeError(str(exc)) if not entries: response: Dict[str, Any] = { "hash": hash_hex, "metadata": {}, "tags": [], "warnings": [f"No Hydrus metadata for {hash_hex or file_ids}"], "error": "not_found", } if file_ids: response["file_id"] = file_ids[0] return response entry = entries[0] if not hash_hex: entry_hash = entry.get("hash") if isinstance(entry_hash, str) and entry_hash: hash_hex = entry_hash hashes = [hash_hex] summary, primary_tags, service_key, title, clip_time = _summarize_hydrus_entry(entry, prefer_service) is_video, is_audio, is_deleted = _compute_file_flags(entry) has_current_file_service = _has_current_file_service(entry) is_local = bool(entry.get("is_local")) size_bytes = entry.get("size") or entry.get("file_size") filesize_mb = None if isinstance(size_bytes, (int, float)) and size_bytes > 0: filesize_mb = float(size_bytes) / (1024.0 * 1024.0) duration = entry.get("duration") if duration is None and isinstance(entry.get("duration_ms"), (int, float)): duration = float(entry["duration_ms"]) / 1000.0 warnings_list: List[str] = [] if not primary_tags: warnings_list.append("No tags returned for preferred service") relationships = None relationship_metadata: Dict[str, Dict[str, Any]] = {} if include_relationships and hash_hex: try: rel_spec = HydrusRequestSpec( method="GET", endpoint="/manage_file_relationships/get_file_relationships", query={"hash": hash_hex}, ) relationships = client._perform_request(rel_spec) except HydrusRequestError as exc: warnings_list.append(f"Relationship lookup failed: {exc}") relationships = None if isinstance(relationships, dict): related_hashes: Set[str] = set() _collect_relationship_hashes(relationships, related_hashes) related_hashes.discard(hash_hex) if related_hashes: try: related_entries = _fetch_hydrus_entries( client, sorted(related_hashes), None, False, True ) except HydrusRequestError as exc: warnings_list.append(f"Relationship metadata fetch failed: {exc}") else: for rel_entry in related_entries: rel_hash = rel_entry.get("hash") if not isinstance(rel_hash, str): continue rel_summary, rel_tags, _, rel_title, rel_clip = _summarize_hydrus_entry(rel_entry, prefer_service) rel_summary["tags"] = rel_tags if rel_title: rel_summary["title"] = rel_title if rel_clip: rel_summary["clip_time"] = rel_clip relationship_metadata[rel_hash] = rel_summary result: Dict[str, Any] = { "hash": entry.get("hash") or hash_hex, "metadata": summary, "tags": primary_tags, "tag_service_key": service_key, "title": title, "clip_time": clip_time, "duration": duration, "filesize_mb": filesize_mb, "is_video": is_video, "is_audio": is_audio, "is_deleted": is_deleted, "is_local": is_local, "has_current_file_service": has_current_file_service, "matched_hash": entry.get("hash") or hash_hex, "swap_recommended": False, } file_id_value = entry.get("file_id") if isinstance(file_id_value, (int, float)): result["file_id"] = int(file_id_value) if relationships is not None: result["relationships"] = relationships if relationship_metadata: result["relationship_metadata"] = relationship_metadata if warnings_list: result["warnings"] = warnings_list return result def fetch_hydrus_metadata_by_url(payload: Dict[str, Any]) -> Dict[str, Any]: raw_url = payload.get("url") or payload.get("source_url") url = str(raw_url or "").strip() if not url: raise ValueError("URL is required to fetch Hydrus metadata by URL") base_url = str(payload.get("api_url") or "").strip() if not base_url: raise ValueError("Hydrus api_url is required") access_key = str(payload.get("access_key") or "").strip() options_raw = payload.get("options") options = options_raw if isinstance(options_raw, dict) else {} timeout = float(options.get("timeout") or 60.0) client = HydrusNetwork(base_url, access_key, timeout) hashes: Optional[List[str]] = None file_ids: Optional[List[int]] = None matched_url = None normalised_reported = None seen: Set[str] = set() queue = deque() for variant in _generate_hydrus_url_variants(url): queue.append(variant) if not queue: queue.append(url) tried_variants: List[str] = [] while queue: candidate = queue.popleft() candidate = str(candidate or "").strip() if not candidate or candidate in seen: continue seen.add(candidate) tried_variants.append(candidate) spec = HydrusRequestSpec( method="GET", endpoint="/add_urls/get_url_files", query={"url": candidate}, ) try: response = client._perform_request(spec) except HydrusRequestError as exc: raise RuntimeError(str(exc)) response_hashes_list: List[str] = [] response_file_ids_list: List[int] = [] if isinstance(response, dict): normalised_value = response.get("normalised_url") if isinstance(normalised_value, str): trimmed = normalised_value.strip() if trimmed: normalised_reported = normalised_reported or trimmed if trimmed not in seen: queue.append(trimmed) for redirect_key in ("redirect_url", "url"): redirect_value = response.get(redirect_key) if isinstance(redirect_value, str): redirect_trimmed = redirect_value.strip() if redirect_trimmed and redirect_trimmed not in seen: queue.append(redirect_trimmed) raw_hashes = response.get("hashes") or response.get("file_hashes") if isinstance(raw_hashes, list): for item in raw_hashes: try: normalized = _normalize_hash(item) except ValueError: continue if normalized: response_hashes_list.append(normalized) raw_ids = response.get("file_ids") or response.get("file_id") if isinstance(raw_ids, list): for item in raw_ids: try: response_file_ids_list.append(int(item)) except (TypeError, ValueError): continue elif raw_ids is not None: try: response_file_ids_list.append(int(raw_ids)) except (TypeError, ValueError): pass statuses = response.get("url_file_statuses") if isinstance(statuses, list): for entry in statuses: if not isinstance(entry, dict): continue status_hash = entry.get("hash") or entry.get("file_hash") if status_hash: try: normalized = _normalize_hash(status_hash) except ValueError: normalized = None if normalized: response_hashes_list.append(normalized) status_id = entry.get("file_id") or entry.get("fileid") if status_id is not None: try: response_file_ids_list.append(int(status_id)) except (TypeError, ValueError): pass if not hashes and response_hashes_list: hashes = response_hashes_list if not file_ids and response_file_ids_list: file_ids = response_file_ids_list if hashes or file_ids: matched_url = candidate break if not hashes and not file_ids: raise RuntimeError( "No Hydrus matches for URL variants: " + ", ".join(tried_variants) ) followup_payload = { "api_url": base_url, "access_key": access_key, "hash": hashes[0] if hashes else None, "file_ids": file_ids, "options": {"timeout": timeout, "minimal": True}, } result = fetch_hydrus_metadata(followup_payload) result["matched_url"] = matched_url or url result["normalised_url"] = normalised_reported or matched_url or url result["tried_urls"] = tried_variants return result def _build_hydrus_context(payload: Dict[str, Any]) -> Tuple["HydrusNetwork", str, str, float, Optional[str]]: base_url = str(payload.get("api_url") or "").strip() if not base_url: raise ValueError("Hydrus api_url is required") access_key = str(payload.get("access_key") or "").strip() options_raw = payload.get("options") options = options_raw if isinstance(options_raw, dict) else {} timeout = float(options.get("timeout") or payload.get("timeout") or 60.0) prefer_service = payload.get("prefer_service_name") or options.get("prefer_service_name") if isinstance(prefer_service, str): prefer_service = prefer_service.strip() or None else: prefer_service = None client = HydrusNetwork(base_url, access_key, timeout) return client, base_url, access_key, timeout, prefer_service def _refetch_hydrus_summary( base_url: str, access_key: str, hash_hex: str, timeout: float, prefer_service: Optional[str] ) -> Dict[str, Any]: payload: Dict[str, Any] = { "hash": hash_hex, "api_url": base_url, "access_key": access_key, "options": { "minimal": True, "include_relationships": False, "timeout": timeout, }, } if prefer_service: payload["options"]["prefer_service_name"] = prefer_service return fetch_hydrus_metadata(payload) def apply_hydrus_tag_mutation( payload: Dict[str, Any], add: Iterable[Any], remove: Iterable[Any] ) -> Dict[str, Any]: client, base_url, access_key, timeout, prefer_service = _build_hydrus_context(payload) hash_hex = _normalize_hash(payload.get("hash")) add_list = [_normalize_tag(tag) for tag in add if _normalize_tag(tag)] remove_list = [_normalize_tag(tag) for tag in remove if _normalize_tag(tag)] if not add_list and not remove_list: raise ValueError("No tag changes supplied") service_key = payload.get("service_key") or payload.get("tag_service_key") summary = None if not service_key: summary = _refetch_hydrus_summary(base_url, access_key, hash_hex, timeout, prefer_service) service_key = summary.get("tag_service_key") if not isinstance(service_key, str) or not service_key: raise RuntimeError("Unable to determine Hydrus tag service key") actions: Dict[str, List[str]] = {} if add_list: actions["0"] = [tag for tag in add_list if tag] if remove_list: actions["1"] = [tag for tag in remove_list if tag] if not actions: raise ValueError("Tag mutation produced no actionable changes") request_payload = { "hashes": [hash_hex], "service_keys_to_actions_to_tags": { service_key: actions, }, } try: tag_spec = HydrusRequestSpec( method="POST", endpoint="/add_tags/add_tags", data=request_payload, ) client._perform_request(tag_spec) except HydrusRequestError as exc: raise RuntimeError(str(exc)) summary_after = _refetch_hydrus_summary(base_url, access_key, hash_hex, timeout, prefer_service) result = dict(summary_after) result["added_tags"] = actions.get("0", []) result["removed_tags"] = actions.get("1", []) result["tag_service_key"] = summary_after.get("tag_service_key") return result