From 3ab122a55de474e6fef80e31f0451947b62b3959 Mon Sep 17 00:00:00 2001 From: Nose Date: Sun, 18 Jan 2026 13:10:31 -0800 Subject: [PATCH] f --- API/HydrusNetwork.py | 177 +++++++++++++++++++++++++++------------- Store/HydrusNetwork.py | 4 +- cmdlet/_shared.py | 2 +- cmdlet/add_file.py | 63 +++++++------- cmdlet/download_file.py | 17 +++- 5 files changed, 175 insertions(+), 88 deletions(-) diff --git a/API/HydrusNetwork.py b/API/HydrusNetwork.py index 6c600dc..0073953 100644 --- a/API/HydrusNetwork.py +++ b/API/HydrusNetwork.py @@ -285,6 +285,10 @@ class HydrusNetwork: # Some endpoints are naturally "missing" sometimes and should not spam logs. if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path": + # Some Hydrus deployments do not expose local file system paths via + # /get_files/file_path. Treat 404 as 'not supported' and let callers + # fall back to HTTP download URLs instead of raising an error. + logger.debug(f"{self._log_prefix()} /get_files/file_path returned 404 (not supported) - caller should fallback to HTTP") return {} logger.error(f"{self._log_prefix()} HTTP {status}: {message}") @@ -1572,8 +1576,8 @@ def is_available(config: dict[str, """Check if Hydrus is available and accessible. Performs a lightweight probe to verify: - - Hydrus URL is configured - - Can connect to Hydrus URL/port + - At least one configured Hydrus instance has URL + API configured + - A TCP connection to that instance's host/port can be established Results are cached per session unless use_cache=False. @@ -1593,16 +1597,22 @@ def is_available(config: dict[str, # Use new config helpers first, fallback to old method from SYS.config import get_hydrus_url, get_hydrus_access_key - url = (get_hydrus_url(config, "home") or "").strip() - if not url: - reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)" - _HYDRUS_AVAILABLE = False - _HYDRUS_UNAVAILABLE_REASON = reason - return False, reason + # Collect candidate instances (prioritize 'home') + store_block = (config or {}).get("store") or {} + hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {} - access_key = get_hydrus_access_key(config, "home") or "" - if not access_key: - reason = "Hydrus access key not configured" + candidate_names: list[str] = [] + if isinstance(hydrus_block, dict) and hydrus_block: + # Prefer 'home' first when present for backwards compatibility + names = list(hydrus_block.keys()) + if "home" in names: + candidate_names = ["home"] + [n for n in names if n != "home"] + else: + candidate_names = names + + # If no configured instances, keep previous behavior for clearer message + if not candidate_names: + reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)" _HYDRUS_AVAILABLE = False _HYDRUS_UNAVAILABLE_REASON = reason return False, reason @@ -1613,37 +1623,46 @@ def is_available(config: dict[str, except (TypeError, ValueError): timeout = 5.0 - try: - # Simple TCP connection test to URL/port - import socket - from urllib.parse import urlparse + import socket + from urllib.parse import urlparse - parsed = urlparse(url) - hostname = parsed.hostname or "localhost" - port = parsed.port or (443 if parsed.scheme == "https" else 80) + errors: list[str] = [] + + for name in candidate_names: + url = (get_hydrus_url(config, name) or "").strip() + access_key = get_hydrus_access_key(config, name) or "" + if not url: + errors.append(f"Hydrus URL not configured for instance '{name}'") + continue + if not access_key: + errors.append(f"Hydrus access key not configured for instance '{name}'") + continue - # Try to connect to the host/port - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(timeout) try: - result = sock.connect_ex((hostname, port)) - if result == 0: - _HYDRUS_AVAILABLE = True - _HYDRUS_UNAVAILABLE_REASON = None - return True, None - else: - reason = f"Cannot connect to {hostname}:{port}" - _HYDRUS_AVAILABLE = False - _HYDRUS_UNAVAILABLE_REASON = reason - return False, reason - finally: - sock.close() + parsed = urlparse(url) + hostname = parsed.hostname or "localhost" + port = parsed.port or (443 if parsed.scheme == "https" else 80) - except Exception as exc: - reason = str(exc) - _HYDRUS_AVAILABLE = False - _HYDRUS_UNAVAILABLE_REASON = reason - return False, reason + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + try: + result = sock.connect_ex((hostname, port)) + if result == 0: + _HYDRUS_AVAILABLE = True + _HYDRUS_UNAVAILABLE_REASON = None + return True, None + errors.append(f"Cannot connect to {hostname}:{port} (instance '{name}')") + finally: + sock.close() + except Exception as exc: + errors.append(str(exc)) + continue + + # No candidate succeeded + _HYDRUS_AVAILABLE = False + reason = "; ".join(errors[:3]) if errors else "No Hydrus instances configured with URL and API" + _HYDRUS_UNAVAILABLE_REASON = reason + return False, reason def is_hydrus_available(config: dict[str, Any]) -> bool: @@ -1659,55 +1678,101 @@ def is_hydrus_available(config: dict[str, Any]) -> bool: return available -def get_client(config: dict[str, Any]) -> HydrusNetwork: +def get_client(config: dict[str, Any], instance_name: str | None = None) -> HydrusNetwork: """Create and return a Hydrus client. + If `instance_name` is provided, return a client for that named instance. + If omitted, prefer the 'home' instance when configured, otherwise fall back + to the first configured Hydrus instance found in `config['store']['hydrusnetwork']`. + Uses access-key authentication by default (no session key acquisition). A session key may still be acquired explicitly by calling `HydrusNetwork.ensure_session_key()`. Args: config: Configuration dict with Hydrus settings + instance_name: Optional named Hydrus instance to use (e.g. 'rpi') Returns: - HydrusClient instance + HydrusNetwork client instance Raises: RuntimeError: If Hydrus is not configured or unavailable """ - # Check availability first - if unavailable, raise immediately + # Perform a lightweight availability check first - this checks any configured instance. available, reason = is_available(config) - if not available: - raise RuntimeError(f"Hydrus is unavailable: {reason}") + # We will still attempt to instantiate a client for a configured instance even if the + # availability probe reported unreachable (this keeps behavior resilient in mixed + # network setups), but if no configured instance exists we'll raise below. from SYS.config import get_hydrus_url, get_hydrus_access_key - # Use new config helpers - hydrus_url = (get_hydrus_url(config, "home") or "").strip() - if not hydrus_url: - raise RuntimeError( - "Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)" - ) + chosen_instance: str | None = None + + if instance_name: + chosen_instance = str(instance_name).strip() + # Validate existence of configuration + url_candidate = (get_hydrus_url(config, chosen_instance) or "").strip() + if not url_candidate: + raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'") + access_key_candidate = get_hydrus_access_key(config, chosen_instance) or "" + if not access_key_candidate: + raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'") + else: + # Determine candidate instances from config and prefer 'home' when present + store_block = (config or {}).get("store") or {} + hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {} + candidate_names: list[str] = [] + if isinstance(hydrus_block, dict) and hydrus_block: + names = list(hydrus_block.keys()) + if "home" in names: + candidate_names = ["home"] + [n for n in names if n != "home"] + else: + candidate_names = names + + # Try to pick the first instance with URL+API configured + for name in candidate_names: + url_candidate = (get_hydrus_url(config, name) or "").strip() + access_key_candidate = get_hydrus_access_key(config, name) or "" + if url_candidate and access_key_candidate: + chosen_instance = name + break + + # If nothing suitable found in config, fall back to 'home' behavior (for backwards compatibility) + if chosen_instance is None: + hydrus_url = (get_hydrus_url(config, "home") or "").strip() + if not hydrus_url: + raise RuntimeError( + "Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)" + ) + chosen_instance = "home" + + # Now we have a chosen instance name; resolve its URL and access key (these validations are defensive) + hydrus_url = (get_hydrus_url(config, chosen_instance) or "").strip() + access_key = get_hydrus_access_key(config, chosen_instance) or "" + + if not hydrus_url: + raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'") + if not access_key: + raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'") - access_key = get_hydrus_access_key(config, "home") or "" timeout_raw = config.get("HydrusNetwork_Request_Timeout") try: timeout = float(timeout_raw) if timeout_raw is not None else 60.0 except (TypeError, ValueError): timeout = 60.0 - # Create cache key from URL and access key - cache_key = f"{hydrus_url}#{access_key}" + # Create cache key from URL, access key, and instance name + cache_key = f"{hydrus_url}#{access_key}#{chosen_instance}" # Check if we have a cached client if cache_key in _hydrus_client_cache: return _hydrus_client_cache[cache_key] - # Create new client - client = HydrusNetwork(hydrus_url, access_key, timeout) - - # Cache the client + # Create new client and cache it + client = HydrusNetwork(hydrus_url, access_key, timeout, instance_name=chosen_instance) _hydrus_client_cache[cache_key] = client + return client return client diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index ac39ddd..019e83f 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -362,8 +362,8 @@ class HydrusNetwork(Store): debug( f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}" ) - except Exception: - pass + except Exception as exc: + debug(f"{self._log_prefix()} metadata fetch failed: {exc}") # If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'. # This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files. diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 598963c..0e3a202 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -3612,7 +3612,7 @@ def check_url_exists_in_storage( if client is None: continue if not hydrus_available: - debug("Bulk URL preflight: hydrus availability check failed; attempting best-effort lookup") + debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup") if _timed_out("hydrus scan"): return True diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index fa0c66f..7988c5f 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -553,7 +553,7 @@ class Add_File(Cmdlet): item, path_arg, pipe_obj, config, store_instance=storage_registry ) debug( - f"[add-file] RESOLVED source: path={media_path}, hash={file_hash[:12] if file_hash else 'N/A'}..." + f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..." ) if not media_path: failures += 1 @@ -1112,8 +1112,8 @@ class Add_File(Cmdlet): if dl_path and dl_path.exists(): pipe_obj.path = str(dl_path) return dl_path, str(r_hash), tmp_dir - except Exception: - pass + except Exception as exc: + debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}") # PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result) candidate: Optional[Path] = None @@ -1148,13 +1148,13 @@ class Add_File(Cmdlet): return None, None, None @staticmethod - def _scan_directory_for_files(directory: Path) -> List[Dict[str, Any]]: + def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]: """Scan a directory for supported media files and return list of file info dicts. Each dict contains: - path: Path object - name: filename - - hash: sha256 hash + - hash: sha256 hash (or None if compute_hash=False) - size: file size in bytes - ext: file extension """ @@ -1172,12 +1172,15 @@ class Add_File(Cmdlet): if ext not in SUPPORTED_MEDIA_EXTENSIONS: continue - # Compute hash - try: - file_hash = sha256_file(item) - except Exception as exc: - debug(f"Failed to hash {item}: {exc}") - continue + file_hash = None + # Compute hash if requested (computing can be expensive for large dirs) + if compute_hash: + try: + file_hash = sha256_file(item) + except Exception as exc: + debug(f"Failed to hash {item}: {exc}") + # If hashing is required, skip this file; otherwise include without hash + continue # Get file size try: @@ -2046,10 +2049,15 @@ class Add_File(Cmdlet): except Exception: stored_path = None + # Compute canonical hash value for downstream use (defensive against non-string returns). + if isinstance(file_identifier, str) and len(file_identifier) == 64: + chosen_hash = file_identifier + else: + chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown") + Add_File._update_pipe_object_destination( pipe_obj, - hash_value=file_identifier - if len(file_identifier) == 64 else f_hash or "unknown", + hash_value=chosen_hash, store=backend_name, path=stored_path, tag=tags, @@ -2061,10 +2069,7 @@ class Add_File(Cmdlet): # Emit a search-file-like payload for consistent tables and natural piping. # Keep hash/store for downstream commands (get-tag, get-file, etc.). - resolved_hash = ( - file_identifier if len(file_identifier) == 64 else - (f_hash or file_identifier or "unknown") - ) + resolved_hash = chosen_hash if hydrus_like_backend and tags: try: @@ -2433,19 +2438,21 @@ class Add_File(Cmdlet): @staticmethod def _cleanup_after_success(media_path: Path, delete_source: bool): - if not delete_source: - return - - # Check if it's a temp file that should always be deleted + # Determine whether this is a temporary merge/tracking file which should be + # deleted even when delete_source is False. is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name - if delete_source or is_temp_merge: - ##log(f"Deleting source file...", file=sys.stderr) - try: - media_path.unlink() - Add_File._cleanup_sidecar_files(media_path) - except Exception as exc: - log(f"⚠️ Could not delete file: {exc}", file=sys.stderr) + # If neither explicit delete was requested nor this looks like a temp-merge, + # avoid deleting the source file. + if not delete_source and not is_temp_merge: + return + + # Attempt deletion (best-effort) + try: + media_path.unlink() + Add_File._cleanup_sidecar_files(media_path) + except Exception as exc: + log(f"⚠️ Could not delete file: {exc}", file=sys.stderr) @staticmethod def _cleanup_sidecar_files(media_path: Path): diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index a1fd59b..55c0c10 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -951,7 +951,22 @@ class Download_File(Cmdlet): debug(f"[download-file] Initializing storage interface...") storage = Store(config=config or {}, suppress_debug=True) hydrus_available = bool(is_hydrus_available(config or {})) - + + # If any Hydrus store backend was successfully initialized in the Store + # registry, consider Hydrus available even if the global probe failed. + try: + from Store.HydrusNetwork import HydrusNetwork as _HydrusStoreClass + for bn in storage.list_backends(): + try: + backend = storage[bn] + if isinstance(backend, _HydrusStoreClass): + hydrus_available = True + break + except Exception: + continue + except Exception: + pass + if isinstance(config, dict): config["_storage_cache"] = (storage, hydrus_available) except Exception as e: