f
This commit is contained in:
@@ -285,6 +285,10 @@ class HydrusNetwork:
|
||||
|
||||
# Some endpoints are naturally "missing" sometimes and should not spam logs.
|
||||
if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path":
|
||||
# Some Hydrus deployments do not expose local file system paths via
|
||||
# /get_files/file_path. Treat 404 as 'not supported' and let callers
|
||||
# fall back to HTTP download URLs instead of raising an error.
|
||||
logger.debug(f"{self._log_prefix()} /get_files/file_path returned 404 (not supported) - caller should fallback to HTTP")
|
||||
return {}
|
||||
|
||||
logger.error(f"{self._log_prefix()} HTTP {status}: {message}")
|
||||
@@ -1572,8 +1576,8 @@ def is_available(config: dict[str,
|
||||
"""Check if Hydrus is available and accessible.
|
||||
|
||||
Performs a lightweight probe to verify:
|
||||
- Hydrus URL is configured
|
||||
- Can connect to Hydrus URL/port
|
||||
- At least one configured Hydrus instance has URL + API configured
|
||||
- A TCP connection to that instance's host/port can be established
|
||||
|
||||
Results are cached per session unless use_cache=False.
|
||||
|
||||
@@ -1593,16 +1597,22 @@ def is_available(config: dict[str,
|
||||
# Use new config helpers first, fallback to old method
|
||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||
|
||||
url = (get_hydrus_url(config, "home") or "").strip()
|
||||
if not url:
|
||||
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||
_HYDRUS_AVAILABLE = False
|
||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||
return False, reason
|
||||
# Collect candidate instances (prioritize 'home')
|
||||
store_block = (config or {}).get("store") or {}
|
||||
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
|
||||
|
||||
access_key = get_hydrus_access_key(config, "home") or ""
|
||||
if not access_key:
|
||||
reason = "Hydrus access key not configured"
|
||||
candidate_names: list[str] = []
|
||||
if isinstance(hydrus_block, dict) and hydrus_block:
|
||||
# Prefer 'home' first when present for backwards compatibility
|
||||
names = list(hydrus_block.keys())
|
||||
if "home" in names:
|
||||
candidate_names = ["home"] + [n for n in names if n != "home"]
|
||||
else:
|
||||
candidate_names = names
|
||||
|
||||
# If no configured instances, keep previous behavior for clearer message
|
||||
if not candidate_names:
|
||||
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||
_HYDRUS_AVAILABLE = False
|
||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||
return False, reason
|
||||
@@ -1613,37 +1623,46 @@ def is_available(config: dict[str,
|
||||
except (TypeError, ValueError):
|
||||
timeout = 5.0
|
||||
|
||||
try:
|
||||
# Simple TCP connection test to URL/port
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or "localhost"
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
errors: list[str] = []
|
||||
|
||||
for name in candidate_names:
|
||||
url = (get_hydrus_url(config, name) or "").strip()
|
||||
access_key = get_hydrus_access_key(config, name) or ""
|
||||
if not url:
|
||||
errors.append(f"Hydrus URL not configured for instance '{name}'")
|
||||
continue
|
||||
if not access_key:
|
||||
errors.append(f"Hydrus access key not configured for instance '{name}'")
|
||||
continue
|
||||
|
||||
# Try to connect to the host/port
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
result = sock.connect_ex((hostname, port))
|
||||
if result == 0:
|
||||
_HYDRUS_AVAILABLE = True
|
||||
_HYDRUS_UNAVAILABLE_REASON = None
|
||||
return True, None
|
||||
else:
|
||||
reason = f"Cannot connect to {hostname}:{port}"
|
||||
_HYDRUS_AVAILABLE = False
|
||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||
return False, reason
|
||||
finally:
|
||||
sock.close()
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or "localhost"
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
|
||||
except Exception as exc:
|
||||
reason = str(exc)
|
||||
_HYDRUS_AVAILABLE = False
|
||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||
return False, reason
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
result = sock.connect_ex((hostname, port))
|
||||
if result == 0:
|
||||
_HYDRUS_AVAILABLE = True
|
||||
_HYDRUS_UNAVAILABLE_REASON = None
|
||||
return True, None
|
||||
errors.append(f"Cannot connect to {hostname}:{port} (instance '{name}')")
|
||||
finally:
|
||||
sock.close()
|
||||
except Exception as exc:
|
||||
errors.append(str(exc))
|
||||
continue
|
||||
|
||||
# No candidate succeeded
|
||||
_HYDRUS_AVAILABLE = False
|
||||
reason = "; ".join(errors[:3]) if errors else "No Hydrus instances configured with URL and API"
|
||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||
return False, reason
|
||||
|
||||
|
||||
def is_hydrus_available(config: dict[str, Any]) -> bool:
|
||||
@@ -1659,55 +1678,101 @@ def is_hydrus_available(config: dict[str, Any]) -> bool:
|
||||
return available
|
||||
|
||||
|
||||
def get_client(config: dict[str, Any]) -> HydrusNetwork:
|
||||
def get_client(config: dict[str, Any], instance_name: str | None = None) -> HydrusNetwork:
|
||||
"""Create and return a Hydrus client.
|
||||
|
||||
If `instance_name` is provided, return a client for that named instance.
|
||||
If omitted, prefer the 'home' instance when configured, otherwise fall back
|
||||
to the first configured Hydrus instance found in `config['store']['hydrusnetwork']`.
|
||||
|
||||
Uses access-key authentication by default (no session key acquisition).
|
||||
A session key may still be acquired explicitly by calling
|
||||
`HydrusNetwork.ensure_session_key()`.
|
||||
|
||||
Args:
|
||||
config: Configuration dict with Hydrus settings
|
||||
instance_name: Optional named Hydrus instance to use (e.g. 'rpi')
|
||||
|
||||
Returns:
|
||||
HydrusClient instance
|
||||
HydrusNetwork client instance
|
||||
|
||||
Raises:
|
||||
RuntimeError: If Hydrus is not configured or unavailable
|
||||
"""
|
||||
# Check availability first - if unavailable, raise immediately
|
||||
# Perform a lightweight availability check first - this checks any configured instance.
|
||||
available, reason = is_available(config)
|
||||
if not available:
|
||||
raise RuntimeError(f"Hydrus is unavailable: {reason}")
|
||||
# We will still attempt to instantiate a client for a configured instance even if the
|
||||
# availability probe reported unreachable (this keeps behavior resilient in mixed
|
||||
# network setups), but if no configured instance exists we'll raise below.
|
||||
|
||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||
|
||||
# Use new config helpers
|
||||
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
|
||||
if not hydrus_url:
|
||||
raise RuntimeError(
|
||||
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||
)
|
||||
chosen_instance: str | None = None
|
||||
|
||||
if instance_name:
|
||||
chosen_instance = str(instance_name).strip()
|
||||
# Validate existence of configuration
|
||||
url_candidate = (get_hydrus_url(config, chosen_instance) or "").strip()
|
||||
if not url_candidate:
|
||||
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
|
||||
access_key_candidate = get_hydrus_access_key(config, chosen_instance) or ""
|
||||
if not access_key_candidate:
|
||||
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
|
||||
else:
|
||||
# Determine candidate instances from config and prefer 'home' when present
|
||||
store_block = (config or {}).get("store") or {}
|
||||
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
|
||||
candidate_names: list[str] = []
|
||||
if isinstance(hydrus_block, dict) and hydrus_block:
|
||||
names = list(hydrus_block.keys())
|
||||
if "home" in names:
|
||||
candidate_names = ["home"] + [n for n in names if n != "home"]
|
||||
else:
|
||||
candidate_names = names
|
||||
|
||||
# Try to pick the first instance with URL+API configured
|
||||
for name in candidate_names:
|
||||
url_candidate = (get_hydrus_url(config, name) or "").strip()
|
||||
access_key_candidate = get_hydrus_access_key(config, name) or ""
|
||||
if url_candidate and access_key_candidate:
|
||||
chosen_instance = name
|
||||
break
|
||||
|
||||
# If nothing suitable found in config, fall back to 'home' behavior (for backwards compatibility)
|
||||
if chosen_instance is None:
|
||||
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
|
||||
if not hydrus_url:
|
||||
raise RuntimeError(
|
||||
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||
)
|
||||
chosen_instance = "home"
|
||||
|
||||
# Now we have a chosen instance name; resolve its URL and access key (these validations are defensive)
|
||||
hydrus_url = (get_hydrus_url(config, chosen_instance) or "").strip()
|
||||
access_key = get_hydrus_access_key(config, chosen_instance) or ""
|
||||
|
||||
if not hydrus_url:
|
||||
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
|
||||
if not access_key:
|
||||
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
|
||||
|
||||
access_key = get_hydrus_access_key(config, "home") or ""
|
||||
timeout_raw = config.get("HydrusNetwork_Request_Timeout")
|
||||
try:
|
||||
timeout = float(timeout_raw) if timeout_raw is not None else 60.0
|
||||
except (TypeError, ValueError):
|
||||
timeout = 60.0
|
||||
|
||||
# Create cache key from URL and access key
|
||||
cache_key = f"{hydrus_url}#{access_key}"
|
||||
# Create cache key from URL, access key, and instance name
|
||||
cache_key = f"{hydrus_url}#{access_key}#{chosen_instance}"
|
||||
|
||||
# Check if we have a cached client
|
||||
if cache_key in _hydrus_client_cache:
|
||||
return _hydrus_client_cache[cache_key]
|
||||
|
||||
# Create new client
|
||||
client = HydrusNetwork(hydrus_url, access_key, timeout)
|
||||
|
||||
# Cache the client
|
||||
# Create new client and cache it
|
||||
client = HydrusNetwork(hydrus_url, access_key, timeout, instance_name=chosen_instance)
|
||||
_hydrus_client_cache[cache_key] = client
|
||||
return client
|
||||
|
||||
return client
|
||||
|
||||
|
||||
@@ -362,8 +362,8 @@ class HydrusNetwork(Store):
|
||||
debug(
|
||||
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
debug(f"{self._log_prefix()} metadata fetch failed: {exc}")
|
||||
|
||||
# If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'.
|
||||
# This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files.
|
||||
|
||||
@@ -3612,7 +3612,7 @@ def check_url_exists_in_storage(
|
||||
if client is None:
|
||||
continue
|
||||
if not hydrus_available:
|
||||
debug("Bulk URL preflight: hydrus availability check failed; attempting best-effort lookup")
|
||||
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
|
||||
|
||||
if _timed_out("hydrus scan"):
|
||||
return True
|
||||
|
||||
@@ -553,7 +553,7 @@ class Add_File(Cmdlet):
|
||||
item, path_arg, pipe_obj, config, store_instance=storage_registry
|
||||
)
|
||||
debug(
|
||||
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash[:12] if file_hash else 'N/A'}..."
|
||||
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
|
||||
)
|
||||
if not media_path:
|
||||
failures += 1
|
||||
@@ -1112,8 +1112,8 @@ class Add_File(Cmdlet):
|
||||
if dl_path and dl_path.exists():
|
||||
pipe_obj.path = str(dl_path)
|
||||
return dl_path, str(r_hash), tmp_dir
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}")
|
||||
|
||||
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
|
||||
candidate: Optional[Path] = None
|
||||
@@ -1148,13 +1148,13 @@ class Add_File(Cmdlet):
|
||||
return None, None, None
|
||||
|
||||
@staticmethod
|
||||
def _scan_directory_for_files(directory: Path) -> List[Dict[str, Any]]:
|
||||
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
|
||||
"""Scan a directory for supported media files and return list of file info dicts.
|
||||
|
||||
Each dict contains:
|
||||
- path: Path object
|
||||
- name: filename
|
||||
- hash: sha256 hash
|
||||
- hash: sha256 hash (or None if compute_hash=False)
|
||||
- size: file size in bytes
|
||||
- ext: file extension
|
||||
"""
|
||||
@@ -1172,12 +1172,15 @@ class Add_File(Cmdlet):
|
||||
if ext not in SUPPORTED_MEDIA_EXTENSIONS:
|
||||
continue
|
||||
|
||||
# Compute hash
|
||||
try:
|
||||
file_hash = sha256_file(item)
|
||||
except Exception as exc:
|
||||
debug(f"Failed to hash {item}: {exc}")
|
||||
continue
|
||||
file_hash = None
|
||||
# Compute hash if requested (computing can be expensive for large dirs)
|
||||
if compute_hash:
|
||||
try:
|
||||
file_hash = sha256_file(item)
|
||||
except Exception as exc:
|
||||
debug(f"Failed to hash {item}: {exc}")
|
||||
# If hashing is required, skip this file; otherwise include without hash
|
||||
continue
|
||||
|
||||
# Get file size
|
||||
try:
|
||||
@@ -2046,10 +2049,15 @@ class Add_File(Cmdlet):
|
||||
except Exception:
|
||||
stored_path = None
|
||||
|
||||
# Compute canonical hash value for downstream use (defensive against non-string returns).
|
||||
if isinstance(file_identifier, str) and len(file_identifier) == 64:
|
||||
chosen_hash = file_identifier
|
||||
else:
|
||||
chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown")
|
||||
|
||||
Add_File._update_pipe_object_destination(
|
||||
pipe_obj,
|
||||
hash_value=file_identifier
|
||||
if len(file_identifier) == 64 else f_hash or "unknown",
|
||||
hash_value=chosen_hash,
|
||||
store=backend_name,
|
||||
path=stored_path,
|
||||
tag=tags,
|
||||
@@ -2061,10 +2069,7 @@ class Add_File(Cmdlet):
|
||||
|
||||
# Emit a search-file-like payload for consistent tables and natural piping.
|
||||
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
|
||||
resolved_hash = (
|
||||
file_identifier if len(file_identifier) == 64 else
|
||||
(f_hash or file_identifier or "unknown")
|
||||
)
|
||||
resolved_hash = chosen_hash
|
||||
|
||||
if hydrus_like_backend and tags:
|
||||
try:
|
||||
@@ -2433,19 +2438,21 @@ class Add_File(Cmdlet):
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_after_success(media_path: Path, delete_source: bool):
|
||||
if not delete_source:
|
||||
return
|
||||
|
||||
# Check if it's a temp file that should always be deleted
|
||||
# Determine whether this is a temporary merge/tracking file which should be
|
||||
# deleted even when delete_source is False.
|
||||
is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name
|
||||
|
||||
if delete_source or is_temp_merge:
|
||||
##log(f"Deleting source file...", file=sys.stderr)
|
||||
try:
|
||||
media_path.unlink()
|
||||
Add_File._cleanup_sidecar_files(media_path)
|
||||
except Exception as exc:
|
||||
log(f"⚠️ Could not delete file: {exc}", file=sys.stderr)
|
||||
# If neither explicit delete was requested nor this looks like a temp-merge,
|
||||
# avoid deleting the source file.
|
||||
if not delete_source and not is_temp_merge:
|
||||
return
|
||||
|
||||
# Attempt deletion (best-effort)
|
||||
try:
|
||||
media_path.unlink()
|
||||
Add_File._cleanup_sidecar_files(media_path)
|
||||
except Exception as exc:
|
||||
log(f"⚠️ Could not delete file: {exc}", file=sys.stderr)
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_sidecar_files(media_path: Path):
|
||||
|
||||
@@ -951,7 +951,22 @@ class Download_File(Cmdlet):
|
||||
debug(f"[download-file] Initializing storage interface...")
|
||||
storage = Store(config=config or {}, suppress_debug=True)
|
||||
hydrus_available = bool(is_hydrus_available(config or {}))
|
||||
|
||||
|
||||
# If any Hydrus store backend was successfully initialized in the Store
|
||||
# registry, consider Hydrus available even if the global probe failed.
|
||||
try:
|
||||
from Store.HydrusNetwork import HydrusNetwork as _HydrusStoreClass
|
||||
for bn in storage.list_backends():
|
||||
try:
|
||||
backend = storage[bn]
|
||||
if isinstance(backend, _HydrusStoreClass):
|
||||
hydrus_available = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(config, dict):
|
||||
config["_storage_cache"] = (storage, hydrus_available)
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user