f
This commit is contained in:
@@ -285,6 +285,10 @@ class HydrusNetwork:
|
|||||||
|
|
||||||
# Some endpoints are naturally "missing" sometimes and should not spam logs.
|
# Some endpoints are naturally "missing" sometimes and should not spam logs.
|
||||||
if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path":
|
if status == 404 and spec.endpoint.rstrip("/") == "/get_files/file_path":
|
||||||
|
# Some Hydrus deployments do not expose local file system paths via
|
||||||
|
# /get_files/file_path. Treat 404 as 'not supported' and let callers
|
||||||
|
# fall back to HTTP download URLs instead of raising an error.
|
||||||
|
logger.debug(f"{self._log_prefix()} /get_files/file_path returned 404 (not supported) - caller should fallback to HTTP")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
logger.error(f"{self._log_prefix()} HTTP {status}: {message}")
|
logger.error(f"{self._log_prefix()} HTTP {status}: {message}")
|
||||||
@@ -1572,8 +1576,8 @@ def is_available(config: dict[str,
|
|||||||
"""Check if Hydrus is available and accessible.
|
"""Check if Hydrus is available and accessible.
|
||||||
|
|
||||||
Performs a lightweight probe to verify:
|
Performs a lightweight probe to verify:
|
||||||
- Hydrus URL is configured
|
- At least one configured Hydrus instance has URL + API configured
|
||||||
- Can connect to Hydrus URL/port
|
- A TCP connection to that instance's host/port can be established
|
||||||
|
|
||||||
Results are cached per session unless use_cache=False.
|
Results are cached per session unless use_cache=False.
|
||||||
|
|
||||||
@@ -1593,16 +1597,22 @@ def is_available(config: dict[str,
|
|||||||
# Use new config helpers first, fallback to old method
|
# Use new config helpers first, fallback to old method
|
||||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||||
|
|
||||||
url = (get_hydrus_url(config, "home") or "").strip()
|
# Collect candidate instances (prioritize 'home')
|
||||||
if not url:
|
store_block = (config or {}).get("store") or {}
|
||||||
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
|
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
|
||||||
_HYDRUS_AVAILABLE = False
|
|
||||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
|
||||||
return False, reason
|
|
||||||
|
|
||||||
access_key = get_hydrus_access_key(config, "home") or ""
|
candidate_names: list[str] = []
|
||||||
if not access_key:
|
if isinstance(hydrus_block, dict) and hydrus_block:
|
||||||
reason = "Hydrus access key not configured"
|
# Prefer 'home' first when present for backwards compatibility
|
||||||
|
names = list(hydrus_block.keys())
|
||||||
|
if "home" in names:
|
||||||
|
candidate_names = ["home"] + [n for n in names if n != "home"]
|
||||||
|
else:
|
||||||
|
candidate_names = names
|
||||||
|
|
||||||
|
# If no configured instances, keep previous behavior for clearer message
|
||||||
|
if not candidate_names:
|
||||||
|
reason = "Hydrus URL not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||||
_HYDRUS_AVAILABLE = False
|
_HYDRUS_AVAILABLE = False
|
||||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||||
return False, reason
|
return False, reason
|
||||||
@@ -1613,16 +1623,26 @@ def is_available(config: dict[str,
|
|||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
timeout = 5.0
|
timeout = 5.0
|
||||||
|
|
||||||
try:
|
|
||||||
# Simple TCP connection test to URL/port
|
|
||||||
import socket
|
import socket
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
errors: list[str] = []
|
||||||
|
|
||||||
|
for name in candidate_names:
|
||||||
|
url = (get_hydrus_url(config, name) or "").strip()
|
||||||
|
access_key = get_hydrus_access_key(config, name) or ""
|
||||||
|
if not url:
|
||||||
|
errors.append(f"Hydrus URL not configured for instance '{name}'")
|
||||||
|
continue
|
||||||
|
if not access_key:
|
||||||
|
errors.append(f"Hydrus access key not configured for instance '{name}'")
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
parsed = urlparse(url)
|
parsed = urlparse(url)
|
||||||
hostname = parsed.hostname or "localhost"
|
hostname = parsed.hostname or "localhost"
|
||||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||||
|
|
||||||
# Try to connect to the host/port
|
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
sock.settimeout(timeout)
|
sock.settimeout(timeout)
|
||||||
try:
|
try:
|
||||||
@@ -1631,17 +1651,16 @@ def is_available(config: dict[str,
|
|||||||
_HYDRUS_AVAILABLE = True
|
_HYDRUS_AVAILABLE = True
|
||||||
_HYDRUS_UNAVAILABLE_REASON = None
|
_HYDRUS_UNAVAILABLE_REASON = None
|
||||||
return True, None
|
return True, None
|
||||||
else:
|
errors.append(f"Cannot connect to {hostname}:{port} (instance '{name}')")
|
||||||
reason = f"Cannot connect to {hostname}:{port}"
|
|
||||||
_HYDRUS_AVAILABLE = False
|
|
||||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
|
||||||
return False, reason
|
|
||||||
finally:
|
finally:
|
||||||
sock.close()
|
sock.close()
|
||||||
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
reason = str(exc)
|
errors.append(str(exc))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# No candidate succeeded
|
||||||
_HYDRUS_AVAILABLE = False
|
_HYDRUS_AVAILABLE = False
|
||||||
|
reason = "; ".join(errors[:3]) if errors else "No Hydrus instances configured with URL and API"
|
||||||
_HYDRUS_UNAVAILABLE_REASON = reason
|
_HYDRUS_UNAVAILABLE_REASON = reason
|
||||||
return False, reason
|
return False, reason
|
||||||
|
|
||||||
@@ -1659,55 +1678,101 @@ def is_hydrus_available(config: dict[str, Any]) -> bool:
|
|||||||
return available
|
return available
|
||||||
|
|
||||||
|
|
||||||
def get_client(config: dict[str, Any]) -> HydrusNetwork:
|
def get_client(config: dict[str, Any], instance_name: str | None = None) -> HydrusNetwork:
|
||||||
"""Create and return a Hydrus client.
|
"""Create and return a Hydrus client.
|
||||||
|
|
||||||
|
If `instance_name` is provided, return a client for that named instance.
|
||||||
|
If omitted, prefer the 'home' instance when configured, otherwise fall back
|
||||||
|
to the first configured Hydrus instance found in `config['store']['hydrusnetwork']`.
|
||||||
|
|
||||||
Uses access-key authentication by default (no session key acquisition).
|
Uses access-key authentication by default (no session key acquisition).
|
||||||
A session key may still be acquired explicitly by calling
|
A session key may still be acquired explicitly by calling
|
||||||
`HydrusNetwork.ensure_session_key()`.
|
`HydrusNetwork.ensure_session_key()`.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dict with Hydrus settings
|
config: Configuration dict with Hydrus settings
|
||||||
|
instance_name: Optional named Hydrus instance to use (e.g. 'rpi')
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
HydrusClient instance
|
HydrusNetwork client instance
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
RuntimeError: If Hydrus is not configured or unavailable
|
RuntimeError: If Hydrus is not configured or unavailable
|
||||||
"""
|
"""
|
||||||
# Check availability first - if unavailable, raise immediately
|
# Perform a lightweight availability check first - this checks any configured instance.
|
||||||
available, reason = is_available(config)
|
available, reason = is_available(config)
|
||||||
if not available:
|
# We will still attempt to instantiate a client for a configured instance even if the
|
||||||
raise RuntimeError(f"Hydrus is unavailable: {reason}")
|
# availability probe reported unreachable (this keeps behavior resilient in mixed
|
||||||
|
# network setups), but if no configured instance exists we'll raise below.
|
||||||
|
|
||||||
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
from SYS.config import get_hydrus_url, get_hydrus_access_key
|
||||||
|
|
||||||
# Use new config helpers
|
chosen_instance: str | None = None
|
||||||
|
|
||||||
|
if instance_name:
|
||||||
|
chosen_instance = str(instance_name).strip()
|
||||||
|
# Validate existence of configuration
|
||||||
|
url_candidate = (get_hydrus_url(config, chosen_instance) or "").strip()
|
||||||
|
if not url_candidate:
|
||||||
|
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
|
||||||
|
access_key_candidate = get_hydrus_access_key(config, chosen_instance) or ""
|
||||||
|
if not access_key_candidate:
|
||||||
|
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
|
||||||
|
else:
|
||||||
|
# Determine candidate instances from config and prefer 'home' when present
|
||||||
|
store_block = (config or {}).get("store") or {}
|
||||||
|
hydrus_block = store_block.get("hydrusnetwork") if isinstance(store_block, dict) else {}
|
||||||
|
candidate_names: list[str] = []
|
||||||
|
if isinstance(hydrus_block, dict) and hydrus_block:
|
||||||
|
names = list(hydrus_block.keys())
|
||||||
|
if "home" in names:
|
||||||
|
candidate_names = ["home"] + [n for n in names if n != "home"]
|
||||||
|
else:
|
||||||
|
candidate_names = names
|
||||||
|
|
||||||
|
# Try to pick the first instance with URL+API configured
|
||||||
|
for name in candidate_names:
|
||||||
|
url_candidate = (get_hydrus_url(config, name) or "").strip()
|
||||||
|
access_key_candidate = get_hydrus_access_key(config, name) or ""
|
||||||
|
if url_candidate and access_key_candidate:
|
||||||
|
chosen_instance = name
|
||||||
|
break
|
||||||
|
|
||||||
|
# If nothing suitable found in config, fall back to 'home' behavior (for backwards compatibility)
|
||||||
|
if chosen_instance is None:
|
||||||
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
|
hydrus_url = (get_hydrus_url(config, "home") or "").strip()
|
||||||
if not hydrus_url:
|
if not hydrus_url:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
|
"Hydrus URL is not configured (check config.conf store.hydrusnetwork.home.URL)"
|
||||||
)
|
)
|
||||||
|
chosen_instance = "home"
|
||||||
|
|
||||||
|
# Now we have a chosen instance name; resolve its URL and access key (these validations are defensive)
|
||||||
|
hydrus_url = (get_hydrus_url(config, chosen_instance) or "").strip()
|
||||||
|
access_key = get_hydrus_access_key(config, chosen_instance) or ""
|
||||||
|
|
||||||
|
if not hydrus_url:
|
||||||
|
raise RuntimeError(f"Hydrus URL is not configured for instance '{chosen_instance}'")
|
||||||
|
if not access_key:
|
||||||
|
raise RuntimeError(f"Hydrus access key is not configured for instance '{chosen_instance}'")
|
||||||
|
|
||||||
access_key = get_hydrus_access_key(config, "home") or ""
|
|
||||||
timeout_raw = config.get("HydrusNetwork_Request_Timeout")
|
timeout_raw = config.get("HydrusNetwork_Request_Timeout")
|
||||||
try:
|
try:
|
||||||
timeout = float(timeout_raw) if timeout_raw is not None else 60.0
|
timeout = float(timeout_raw) if timeout_raw is not None else 60.0
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
timeout = 60.0
|
timeout = 60.0
|
||||||
|
|
||||||
# Create cache key from URL and access key
|
# Create cache key from URL, access key, and instance name
|
||||||
cache_key = f"{hydrus_url}#{access_key}"
|
cache_key = f"{hydrus_url}#{access_key}#{chosen_instance}"
|
||||||
|
|
||||||
# Check if we have a cached client
|
# Check if we have a cached client
|
||||||
if cache_key in _hydrus_client_cache:
|
if cache_key in _hydrus_client_cache:
|
||||||
return _hydrus_client_cache[cache_key]
|
return _hydrus_client_cache[cache_key]
|
||||||
|
|
||||||
# Create new client
|
# Create new client and cache it
|
||||||
client = HydrusNetwork(hydrus_url, access_key, timeout)
|
client = HydrusNetwork(hydrus_url, access_key, timeout, instance_name=chosen_instance)
|
||||||
|
|
||||||
# Cache the client
|
|
||||||
_hydrus_client_cache[cache_key] = client
|
_hydrus_client_cache[cache_key] = client
|
||||||
|
return client
|
||||||
|
|
||||||
return client
|
return client
|
||||||
|
|
||||||
|
|||||||
@@ -362,8 +362,8 @@ class HydrusNetwork(Store):
|
|||||||
debug(
|
debug(
|
||||||
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
|
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
pass
|
debug(f"{self._log_prefix()} metadata fetch failed: {exc}")
|
||||||
|
|
||||||
# If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'.
|
# If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'.
|
||||||
# This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files.
|
# This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files.
|
||||||
|
|||||||
@@ -3612,7 +3612,7 @@ def check_url_exists_in_storage(
|
|||||||
if client is None:
|
if client is None:
|
||||||
continue
|
continue
|
||||||
if not hydrus_available:
|
if not hydrus_available:
|
||||||
debug("Bulk URL preflight: hydrus availability check failed; attempting best-effort lookup")
|
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
|
||||||
|
|
||||||
if _timed_out("hydrus scan"):
|
if _timed_out("hydrus scan"):
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -553,7 +553,7 @@ class Add_File(Cmdlet):
|
|||||||
item, path_arg, pipe_obj, config, store_instance=storage_registry
|
item, path_arg, pipe_obj, config, store_instance=storage_registry
|
||||||
)
|
)
|
||||||
debug(
|
debug(
|
||||||
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash[:12] if file_hash else 'N/A'}..."
|
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
|
||||||
)
|
)
|
||||||
if not media_path:
|
if not media_path:
|
||||||
failures += 1
|
failures += 1
|
||||||
@@ -1112,8 +1112,8 @@ class Add_File(Cmdlet):
|
|||||||
if dl_path and dl_path.exists():
|
if dl_path and dl_path.exists():
|
||||||
pipe_obj.path = str(dl_path)
|
pipe_obj.path = str(dl_path)
|
||||||
return dl_path, str(r_hash), tmp_dir
|
return dl_path, str(r_hash), tmp_dir
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
pass
|
debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}")
|
||||||
|
|
||||||
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
|
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
|
||||||
candidate: Optional[Path] = None
|
candidate: Optional[Path] = None
|
||||||
@@ -1148,13 +1148,13 @@ class Add_File(Cmdlet):
|
|||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _scan_directory_for_files(directory: Path) -> List[Dict[str, Any]]:
|
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
|
||||||
"""Scan a directory for supported media files and return list of file info dicts.
|
"""Scan a directory for supported media files and return list of file info dicts.
|
||||||
|
|
||||||
Each dict contains:
|
Each dict contains:
|
||||||
- path: Path object
|
- path: Path object
|
||||||
- name: filename
|
- name: filename
|
||||||
- hash: sha256 hash
|
- hash: sha256 hash (or None if compute_hash=False)
|
||||||
- size: file size in bytes
|
- size: file size in bytes
|
||||||
- ext: file extension
|
- ext: file extension
|
||||||
"""
|
"""
|
||||||
@@ -1172,11 +1172,14 @@ class Add_File(Cmdlet):
|
|||||||
if ext not in SUPPORTED_MEDIA_EXTENSIONS:
|
if ext not in SUPPORTED_MEDIA_EXTENSIONS:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Compute hash
|
file_hash = None
|
||||||
|
# Compute hash if requested (computing can be expensive for large dirs)
|
||||||
|
if compute_hash:
|
||||||
try:
|
try:
|
||||||
file_hash = sha256_file(item)
|
file_hash = sha256_file(item)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
debug(f"Failed to hash {item}: {exc}")
|
debug(f"Failed to hash {item}: {exc}")
|
||||||
|
# If hashing is required, skip this file; otherwise include without hash
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Get file size
|
# Get file size
|
||||||
@@ -2046,10 +2049,15 @@ class Add_File(Cmdlet):
|
|||||||
except Exception:
|
except Exception:
|
||||||
stored_path = None
|
stored_path = None
|
||||||
|
|
||||||
|
# Compute canonical hash value for downstream use (defensive against non-string returns).
|
||||||
|
if isinstance(file_identifier, str) and len(file_identifier) == 64:
|
||||||
|
chosen_hash = file_identifier
|
||||||
|
else:
|
||||||
|
chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown")
|
||||||
|
|
||||||
Add_File._update_pipe_object_destination(
|
Add_File._update_pipe_object_destination(
|
||||||
pipe_obj,
|
pipe_obj,
|
||||||
hash_value=file_identifier
|
hash_value=chosen_hash,
|
||||||
if len(file_identifier) == 64 else f_hash or "unknown",
|
|
||||||
store=backend_name,
|
store=backend_name,
|
||||||
path=stored_path,
|
path=stored_path,
|
||||||
tag=tags,
|
tag=tags,
|
||||||
@@ -2061,10 +2069,7 @@ class Add_File(Cmdlet):
|
|||||||
|
|
||||||
# Emit a search-file-like payload for consistent tables and natural piping.
|
# Emit a search-file-like payload for consistent tables and natural piping.
|
||||||
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
|
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
|
||||||
resolved_hash = (
|
resolved_hash = chosen_hash
|
||||||
file_identifier if len(file_identifier) == 64 else
|
|
||||||
(f_hash or file_identifier or "unknown")
|
|
||||||
)
|
|
||||||
|
|
||||||
if hydrus_like_backend and tags:
|
if hydrus_like_backend and tags:
|
||||||
try:
|
try:
|
||||||
@@ -2433,14 +2438,16 @@ class Add_File(Cmdlet):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _cleanup_after_success(media_path: Path, delete_source: bool):
|
def _cleanup_after_success(media_path: Path, delete_source: bool):
|
||||||
if not delete_source:
|
# Determine whether this is a temporary merge/tracking file which should be
|
||||||
return
|
# deleted even when delete_source is False.
|
||||||
|
|
||||||
# Check if it's a temp file that should always be deleted
|
|
||||||
is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name
|
is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name
|
||||||
|
|
||||||
if delete_source or is_temp_merge:
|
# If neither explicit delete was requested nor this looks like a temp-merge,
|
||||||
##log(f"Deleting source file...", file=sys.stderr)
|
# avoid deleting the source file.
|
||||||
|
if not delete_source and not is_temp_merge:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Attempt deletion (best-effort)
|
||||||
try:
|
try:
|
||||||
media_path.unlink()
|
media_path.unlink()
|
||||||
Add_File._cleanup_sidecar_files(media_path)
|
Add_File._cleanup_sidecar_files(media_path)
|
||||||
|
|||||||
@@ -952,6 +952,21 @@ class Download_File(Cmdlet):
|
|||||||
storage = Store(config=config or {}, suppress_debug=True)
|
storage = Store(config=config or {}, suppress_debug=True)
|
||||||
hydrus_available = bool(is_hydrus_available(config or {}))
|
hydrus_available = bool(is_hydrus_available(config or {}))
|
||||||
|
|
||||||
|
# If any Hydrus store backend was successfully initialized in the Store
|
||||||
|
# registry, consider Hydrus available even if the global probe failed.
|
||||||
|
try:
|
||||||
|
from Store.HydrusNetwork import HydrusNetwork as _HydrusStoreClass
|
||||||
|
for bn in storage.list_backends():
|
||||||
|
try:
|
||||||
|
backend = storage[bn]
|
||||||
|
if isinstance(backend, _HydrusStoreClass):
|
||||||
|
hydrus_available = True
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
if isinstance(config, dict):
|
if isinstance(config, dict):
|
||||||
config["_storage_cache"] = (storage, hydrus_available)
|
config["_storage_cache"] = (storage, hydrus_available)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
Reference in New Issue
Block a user