This commit is contained in:
2026-01-18 13:10:31 -08:00
parent 66e6c6eb72
commit 3ab122a55d
5 changed files with 175 additions and 88 deletions

View File

@@ -285,6 +285,10 @@ class HydrusNetwork:
# Some endpoints are naturally "missing" sometimes and should not spam logs.
if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path":
# Some Hydrus deployments do not expose local file system paths via
# /get_files/file_path. Treat 404 as 'not supported' and let callers
# fall back to HTTP download URLs instead of raising an error.
logger.debug(f"{self._log_prefix()} /get_files/file_path returned 404 (not supported) - caller should fallback to HTTP")
return {}
logger.error(f"{self._log_prefix()} HTTP {status}: {message}")
@@ -1572,8 +1576,8 @@ def is_available(config: dict[str,
"""Check if Hydrus is available and accessible.
Performs a lightweight probe to verify:
- Hydrus URL is configured
- Can connect to Hydrus URL/port
- At least one configured Hydrus instance has URL + API configured
- A TCP connection to that instance's host/port can be established
Results are cached per session unless use_cache=False.
@@ -1593,16 +1597,22 @@ def is_available(config: dict[str,
# Use new config helpers first, fallback to old method
from SYS.config import get_hydrus_url, get_hydrus_access_key
url = (get_hydrus_url(config, "home") or "").strip()
if not url:
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
_HYDRUS_AVAILABLE = False
_HYDRUS_UNAVAILABLE_REASON = reason
return False, reason
# Collect candidate instances (prioritize 'home')
store_block = (config or {}).get("store") or {}
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
access_key = get_hydrus_access_key(config, "home") or ""
if not access_key:
reason = "Hydrus access key not configured"
candidate_names: list[str] = []
if isinstance(hydrus_block, dict) and hydrus_block:
# Prefer 'home' first when present for backwards compatibility
names = list(hydrus_block.keys())
if "home" in names:
candidate_names = ["home"] + [n for n in names if n != "home"]
else:
candidate_names = names
# If no configured instances, keep previous behavior for clearer message
if not candidate_names:
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
_HYDRUS_AVAILABLE = False
_HYDRUS_UNAVAILABLE_REASON = reason
return False, reason
@@ -1613,37 +1623,46 @@ def is_available(config: dict[str,
except (TypeError, ValueError):
timeout = 5.0
try:
# Simple TCP connection test to URL/port
import socket
from urllib.parse import urlparse
import socket
from urllib.parse import urlparse
parsed = urlparse(url)
hostname = parsed.hostname or "localhost"
port = parsed.port or (443 if parsed.scheme == "https" else 80)
errors: list[str] = []
for name in candidate_names:
url = (get_hydrus_url(config, name) or "").strip()
access_key = get_hydrus_access_key(config, name) or ""
if not url:
errors.append(f"Hydrus URL not configured for instance '{name}'")
continue
if not access_key:
errors.append(f"Hydrus access key not configured for instance '{name}'")
continue
# Try to connect to the host/port
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
result = sock.connect_ex((hostname, port))
if result == 0:
_HYDRUS_AVAILABLE = True
_HYDRUS_UNAVAILABLE_REASON = None
return True, None
else:
reason = f"Cannot connect to {hostname}:{port}"
_HYDRUS_AVAILABLE = False
_HYDRUS_UNAVAILABLE_REASON = reason
return False, reason
finally:
sock.close()
parsed = urlparse(url)
hostname = parsed.hostname or "localhost"
port = parsed.port or (443 if parsed.scheme == "https" else 80)
except Exception as exc:
reason = str(exc)
_HYDRUS_AVAILABLE = False
_HYDRUS_UNAVAILABLE_REASON = reason
return False, reason
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
result = sock.connect_ex((hostname, port))
if result == 0:
_HYDRUS_AVAILABLE = True
_HYDRUS_UNAVAILABLE_REASON = None
return True, None
errors.append(f"Cannot connect to {hostname}:{port} (instance '{name}')")
finally:
sock.close()
except Exception as exc:
errors.append(str(exc))
continue
# No candidate succeeded
_HYDRUS_AVAILABLE = False
reason = "; ".join(errors[:3]) if errors else "No Hydrus instances configured with URL and API"
_HYDRUS_UNAVAILABLE_REASON = reason
return False, reason
def is_hydrus_available(config: dict[str, Any]) -> bool:
@@ -1659,55 +1678,101 @@ def is_hydrus_available(config: dict[str, Any]) -> bool:
return available
def get_client(config: dict[str, Any]) -> HydrusNetwork:
def get_client(config: dict[str, Any], instance_name: str | None = None) -> HydrusNetwork:
"""Create and return a Hydrus client.
If `instance_name` is provided, return a client for that named instance.
If omitted, prefer the 'home' instance when configured, otherwise fall back
to the first configured Hydrus instance found in `config['store']['hydrusnetwork']`.
Uses access-key authentication by default (no session key acquisition).
A session key may still be acquired explicitly by calling
`HydrusNetwork.ensure_session_key()`.
Args:
config: Configuration dict with Hydrus settings
instance_name: Optional named Hydrus instance to use (e.g. 'rpi')
Returns:
HydrusClient instance
HydrusNetwork client instance
Raises:
RuntimeError: If Hydrus is not configured or unavailable
"""
# Check availability first - if unavailable, raise immediately
# Perform a lightweight availability check first - this checks any configured instance.
available, reason = is_available(config)
if not available:
raise RuntimeError(f"Hydrus is unavailable: {reason}")
# We will still attempt to instantiate a client for a configured instance even if the
# availability probe reported unreachable (this keeps behavior resilient in mixed
# network setups), but if no configured instance exists we'll raise below.
from SYS.config import get_hydrus_url, get_hydrus_access_key
# Use new config helpers
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
if not hydrus_url:
raise RuntimeError(
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
)
chosen_instance: str | None = None
if instance_name:
chosen_instance = str(instance_name).strip()
# Validate existence of configuration
url_candidate = (get_hydrus_url(config, chosen_instance) or "").strip()
if not url_candidate:
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
access_key_candidate = get_hydrus_access_key(config, chosen_instance) or ""
if not access_key_candidate:
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
else:
# Determine candidate instances from config and prefer 'home' when present
store_block = (config or {}).get("store") or {}
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
candidate_names: list[str] = []
if isinstance(hydrus_block, dict) and hydrus_block:
names = list(hydrus_block.keys())
if "home" in names:
candidate_names = ["home"] + [n for n in names if n != "home"]
else:
candidate_names = names
# Try to pick the first instance with URL+API configured
for name in candidate_names:
url_candidate = (get_hydrus_url(config, name) or "").strip()
access_key_candidate = get_hydrus_access_key(config, name) or ""
if url_candidate and access_key_candidate:
chosen_instance = name
break
# If nothing suitable found in config, fall back to 'home' behavior (for backwards compatibility)
if chosen_instance is None:
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
if not hydrus_url:
raise RuntimeError(
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
)
chosen_instance = "home"
# Now we have a chosen instance name; resolve its URL and access key (these validations are defensive)
hydrus_url = (get_hydrus_url(config, chosen_instance) or "").strip()
access_key = get_hydrus_access_key(config, chosen_instance) or ""
if not hydrus_url:
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
if not access_key:
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
access_key = get_hydrus_access_key(config, "home") or ""
timeout_raw = config.get("HydrusNetwork_Request_Timeout")
try:
timeout = float(timeout_raw) if timeout_raw is not None else 60.0
except (TypeError, ValueError):
timeout = 60.0
# Create cache key from URL and access key
cache_key = f"{hydrus_url}#{access_key}"
# Create cache key from URL, access key, and instance name
cache_key = f"{hydrus_url}#{access_key}#{chosen_instance}"
# Check if we have a cached client
if cache_key in _hydrus_client_cache:
return _hydrus_client_cache[cache_key]
# Create new client
client = HydrusNetwork(hydrus_url, access_key, timeout)
# Cache the client
# Create new client and cache it
client = HydrusNetwork(hydrus_url, access_key, timeout, instance_name=chosen_instance)
_hydrus_client_cache[cache_key] = client
return client
return client

View File

@@ -362,8 +362,8 @@ class HydrusNetwork(Store):
debug(
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
)
except Exception:
pass
except Exception as exc:
debug(f"{self._log_prefix()} metadata fetch failed: {exc}")
# If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'.
# This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files.

View File

@@ -3612,7 +3612,7 @@ def check_url_exists_in_storage(
if client is None:
continue
if not hydrus_available:
debug("Bulk URL preflight: hydrus availability check failed; attempting best-effort lookup")
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
if _timed_out("hydrus scan"):
return True

View File

@@ -553,7 +553,7 @@ class Add_File(Cmdlet):
item, path_arg, pipe_obj, config, store_instance=storage_registry
)
debug(
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash[:12] if file_hash else 'N/A'}..."
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
)
if not media_path:
failures += 1
@@ -1112,8 +1112,8 @@ class Add_File(Cmdlet):
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
return dl_path, str(r_hash), tmp_dir
except Exception:
pass
except Exception as exc:
debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}")
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
candidate: Optional[Path] = None
@@ -1148,13 +1148,13 @@ class Add_File(Cmdlet):
return None, None, None
@staticmethod
def _scan_directory_for_files(directory: Path) -> List[Dict[str, Any]]:
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
"""Scan a directory for supported media files and return list of file info dicts.
Each dict contains:
- path: Path object
- name: filename
- hash: sha256 hash
- hash: sha256 hash (or None if compute_hash=False)
- size: file size in bytes
- ext: file extension
"""
@@ -1172,12 +1172,15 @@ class Add_File(Cmdlet):
if ext not in SUPPORTED_MEDIA_EXTENSIONS:
continue
# Compute hash
try:
file_hash = sha256_file(item)
except Exception as exc:
debug(f"Failed to hash {item}: {exc}")
continue
file_hash = None
# Compute hash if requested (computing can be expensive for large dirs)
if compute_hash:
try:
file_hash = sha256_file(item)
except Exception as exc:
debug(f"Failed to hash {item}: {exc}")
# If hashing is required, skip this file; otherwise include without hash
continue
# Get file size
try:
@@ -2046,10 +2049,15 @@ class Add_File(Cmdlet):
except Exception:
stored_path = None
# Compute canonical hash value for downstream use (defensive against non-string returns).
if isinstance(file_identifier, str) and len(file_identifier) == 64:
chosen_hash = file_identifier
else:
chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown")
Add_File._update_pipe_object_destination(
pipe_obj,
hash_value=file_identifier
if len(file_identifier) == 64 else f_hash or "unknown",
hash_value=chosen_hash,
store=backend_name,
path=stored_path,
tag=tags,
@@ -2061,10 +2069,7 @@ class Add_File(Cmdlet):
# Emit a search-file-like payload for consistent tables and natural piping.
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
resolved_hash = (
file_identifier if len(file_identifier) == 64 else
(f_hash or file_identifier or "unknown")
)
resolved_hash = chosen_hash
if hydrus_like_backend and tags:
try:
@@ -2433,19 +2438,21 @@ class Add_File(Cmdlet):
@staticmethod
def _cleanup_after_success(media_path: Path, delete_source: bool):
if not delete_source:
return
# Check if it's a temp file that should always be deleted
# Determine whether this is a temporary merge/tracking file which should be
# deleted even when delete_source is False.
is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name
if delete_source or is_temp_merge:
##log(f"Deleting source file...", file=sys.stderr)
try:
media_path.unlink()
Add_File._cleanup_sidecar_files(media_path)
except Exception as exc:
log(f"⚠️ Could not delete file: {exc}", file=sys.stderr)
# If neither explicit delete was requested nor this looks like a temp-merge,
# avoid deleting the source file.
if not delete_source and not is_temp_merge:
return
# Attempt deletion (best-effort)
try:
media_path.unlink()
Add_File._cleanup_sidecar_files(media_path)
except Exception as exc:
log(f"⚠️ Could not delete file: {exc}", file=sys.stderr)
@staticmethod
def _cleanup_sidecar_files(media_path: Path):

View File

@@ -951,7 +951,22 @@ class Download_File(Cmdlet):
debug(f"[download-file] Initializing storage interface...")
storage = Store(config=config or {}, suppress_debug=True)
hydrus_available = bool(is_hydrus_available(config or {}))
# If any Hydrus store backend was successfully initialized in the Store
# registry, consider Hydrus available even if the global probe failed.
try:
from Store.HydrusNetwork import HydrusNetwork as _HydrusStoreClass
for bn in storage.list_backends():
try:
backend = storage[bn]
if isinstance(backend, _HydrusStoreClass):
hydrus_available = True
break
except Exception:
continue
except Exception:
pass
if isinstance(config, dict):
config["_storage_cache"] = (storage, hydrus_available)
except Exception as e: