This commit is contained in:
nose
2025-12-16 23:23:43 -08:00
parent 9873280f0e
commit 86918f2ae2
46 changed files with 2277 additions and 1347 deletions

View File

@@ -23,6 +23,10 @@ class HydrusNetwork(Store):
Maintains its own HydrusClient.
"""
def _log_prefix(self) -> str:
store_name = getattr(self, "NAME", None) or "unknown"
return f"[hydrusnetwork:{store_name}]"
def __new__(cls, *args: Any, **kwargs: Any) -> "HydrusNetwork":
instance = super().__new__(cls)
name = kwargs.get("NAME")
@@ -109,7 +113,7 @@ class HydrusNetwork(Store):
raise RuntimeError(f"Hydrus '{self.NAME}' unavailable: {err}") from exc
# Create a persistent client for this instance (auth via access key by default).
self._client = HydrusClient(url=self.URL, access_key=self.API)
self._client = HydrusClient(url=self.URL, access_key=self.API, instance_name=self.NAME)
# Best-effort total count (fast on Hydrus side; does not fetch IDs/hashes).
try:
@@ -129,7 +133,7 @@ class HydrusNetwork(Store):
if isinstance(count_val, int):
self.total_count = count_val
except Exception as exc:
debug(f"Hydrus total count unavailable for '{self.NAME}': {exc}", file=sys.stderr)
debug(f"{self._log_prefix()} total count unavailable: {exc}", file=sys.stderr)
def name(self) -> str:
return self.NAME
@@ -167,7 +171,7 @@ class HydrusNetwork(Store):
try:
# Compute file hash
file_hash = sha256_file(file_path)
debug(f"File hash: {file_hash}")
debug(f"{self._log_prefix()} file hash: {file_hash}")
# Use persistent client with session key
client = self._client
@@ -177,21 +181,42 @@ class HydrusNetwork(Store):
# Check if file already exists in Hydrus
file_exists = False
try:
metadata = client.fetch_file_metadata(hashes=[file_hash])
metadata = client.fetch_file_metadata(
hashes=[file_hash],
include_service_keys_to_tags=False,
include_file_url=False,
include_duration=False,
include_size=False,
include_mime=False,
)
if metadata and isinstance(metadata, dict):
files = metadata.get("metadata", [])
if files:
file_exists = True
log(
f" Duplicate detected - file already in Hydrus with hash: {file_hash}",
file=sys.stderr,
)
metas = metadata.get("metadata", [])
if isinstance(metas, list) and metas:
# Hydrus returns placeholder rows for unknown hashes.
# Only treat as a real duplicate if it has a concrete file_id.
for meta in metas:
if isinstance(meta, dict) and meta.get("file_id") is not None:
file_exists = True
break
if file_exists:
log(
f" Duplicate detected - file already in Hydrus with hash: {file_hash}",
file=sys.stderr,
)
except Exception:
pass
# If Hydrus reports an existing file, it may be in trash. Best-effort restore it to 'my files'.
# This keeps behavior aligned with user expectation: "use API only" and ensure it lands in my files.
if file_exists:
try:
client.undelete_files([file_hash])
except Exception:
pass
# Upload file if not already present
if not file_exists:
log(f"Uploading to Hydrus: {file_path.name}", file=sys.stderr)
log(f"{self._log_prefix()} Uploading: {file_path.name}", file=sys.stderr)
response = client.add_file(file_path)
# Extract hash from response
@@ -207,7 +232,7 @@ class HydrusNetwork(Store):
raise Exception(f"Hydrus response missing file hash: {response}")
file_hash = hydrus_hash
log(f"Hydrus: {file_hash}", file=sys.stderr)
log(f"{self._log_prefix()} hash: {file_hash}", file=sys.stderr)
# Add tags if provided (both for new and existing files)
if tag_list:
@@ -218,27 +243,27 @@ class HydrusNetwork(Store):
service_name = "my tags"
try:
debug(f"Adding {len(tag_list)} tag(s) to Hydrus: {tag_list}")
debug(f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}")
client.add_tag(file_hash, tag_list, service_name)
log(f"Tags added via '{service_name}'", file=sys.stderr)
log(f"{self._log_prefix()} Tags added via '{service_name}'", file=sys.stderr)
except Exception as exc:
log(f"⚠️ Failed to add tags: {exc}", file=sys.stderr)
log(f"{self._log_prefix()} ⚠️ Failed to add tags: {exc}", file=sys.stderr)
# Associate url if provided (both for new and existing files)
if url:
log(f"Associating {len(url)} URL(s) with file", file=sys.stderr)
log(f"{self._log_prefix()} Associating {len(url)} URL(s) with file", file=sys.stderr)
for url in url:
if url:
try:
client.associate_url(file_hash, str(url))
debug(f"Associated URL: {url}")
debug(f"{self._log_prefix()} Associated URL: {url}")
except Exception as exc:
log(f"⚠️ Failed to associate URL {url}: {exc}", file=sys.stderr)
log(f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}", file=sys.stderr)
return file_hash
except Exception as exc:
log(f"❌ Hydrus upload failed: {exc}", file=sys.stderr)
log(f"{self._log_prefix()} upload failed: {exc}", file=sys.stderr)
raise
def search(self, query: str, **kwargs: Any) -> list[Dict[str, Any]]:
@@ -262,7 +287,8 @@ class HydrusNetwork(Store):
if client is None:
raise Exception("Hydrus client unavailable")
debug(f"Searching Hydrus for: {query}")
prefix = self._log_prefix()
debug(f"{prefix} Searching for: {query}")
def _extract_urls(meta_obj: Any) -> list[str]:
if not isinstance(meta_obj, dict):
@@ -446,7 +472,7 @@ class HydrusNetwork(Store):
tags = [query_lower]
if not tags:
debug(f"Found 0 result(s)")
debug(f"{prefix} 0 result(s)")
return []
# Search files with the tags (unless url: search already produced metadata)
@@ -465,7 +491,7 @@ class HydrusNetwork(Store):
hashes = search_result.get("hashes", []) if isinstance(search_result, dict) else []
if not file_ids and not hashes:
debug(f"Found 0 result(s)")
debug(f"{prefix} 0 result(s)")
return []
if file_ids:
@@ -595,7 +621,7 @@ class HydrusNetwork(Store):
"ext": ext,
})
debug(f"Found {len(results)} result(s)")
debug(f"{prefix} {len(results)} result(s)")
return results[:limit]
except Exception as exc:
@@ -611,13 +637,13 @@ class HydrusNetwork(Store):
Only explicit user actions (e.g. the get-file cmdlet) should open files.
"""
debug(f"[HydrusNetwork.get_file] Starting for hash: {file_hash[:12]}...")
debug(f"{self._log_prefix()} get_file: start hash={file_hash[:12]}...")
# Build browser URL with access key
base_url = str(self.URL).rstrip('/')
access_key = str(self.API)
browser_url = f"{base_url}/get_files/file?hash={file_hash}&Hydrus-Client-API-Access-Key={access_key}"
debug(f"[HydrusNetwork.get_file] Returning URL: {browser_url}")
debug(f"{self._log_prefix()} get_file: url={browser_url}")
return browser_url
def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
@@ -632,16 +658,27 @@ class HydrusNetwork(Store):
try:
client = self._client
if not client:
debug("get_metadata: Hydrus client unavailable")
debug(f"{self._log_prefix()} get_metadata: client unavailable")
return None
# Fetch file metadata
payload = client.fetch_file_metadata(hashes=[file_hash], include_service_keys_to_tags=True)
# Fetch file metadata with the fields we need for CLI display.
payload = client.fetch_file_metadata(
hashes=[file_hash],
include_service_keys_to_tags=True,
include_file_url=True,
include_duration=True,
include_size=True,
include_mime=True,
)
if not payload or not payload.get("metadata"):
return None
meta = payload["metadata"][0]
# Hydrus can return placeholder metadata rows for unknown hashes.
if not isinstance(meta, dict) or meta.get("file_id") is None:
return None
# Extract title from tags
title = f"Hydrus_{file_hash[:12]}"
@@ -660,33 +697,109 @@ class HydrusNetwork(Store):
if title != f"Hydrus_{file_hash[:12]}":
break
# Prefer Hydrus-provided extension (e.g. ".webm"); fall back to MIME map if needed.
mime_type = meta.get("mime", "")
ext_raw = meta.get("ext")
ext = str(ext_raw or "").strip().lstrip(".")
if not ext and mime_type:
# Hydrus may return mime as an int enum, or sometimes a human label.
mime_val = meta.get("mime")
filetype_human = meta.get("filetype_human") or meta.get("mime_human") or meta.get("mime_string")
# Determine ext: prefer Hydrus metadata ext, then filetype_human (when it looks like an ext),
# then title suffix, then file path suffix.
ext = str(meta.get("ext") or "").strip().lstrip(".")
if not ext:
ft = str(filetype_human or "").strip().lstrip(".").lower()
if ft and ft != "unknown filetype" and ft.isalnum() and len(ft) <= 8:
# Treat simple labels like "mp4", "m4a", "webm" as extensions.
ext = ft
if not ext and isinstance(title, str) and "." in title:
try:
from SYS.utils_constant import mime_maps
for category in mime_maps.values():
for _ext_key, info in category.items():
if mime_type in info.get("mimes", []):
ext = str(info.get("ext", "")).strip().lstrip(".")
break
if ext:
break
ext = Path(title).suffix.lstrip(".")
except Exception:
ext = ""
if not ext:
try:
path_payload = client.get_file_path(file_hash)
if isinstance(path_payload, dict):
p = path_payload.get("path")
if isinstance(p, str) and p.strip():
ext = Path(p.strip()).suffix.lstrip(".")
except Exception:
ext = ""
# If extension is still unknown, attempt a best-effort lookup from MIME.
def _mime_from_ext(ext_value: str) -> str:
ext_clean = str(ext_value or "").strip().lstrip(".").lower()
if not ext_clean:
return ""
try:
for category in mime_maps.values():
info = category.get(ext_clean)
if isinstance(info, dict):
mimes = info.get("mimes")
if isinstance(mimes, list) and mimes:
first = mimes[0]
return str(first)
except Exception:
return ""
return ""
# Normalize to a MIME string for CLI output.
# Avoid passing through human labels like "unknown filetype".
mime_type = ""
if isinstance(mime_val, str):
candidate = mime_val.strip()
if "/" in candidate and candidate.lower() != "unknown filetype":
mime_type = candidate
if not mime_type and isinstance(filetype_human, str):
candidate = filetype_human.strip()
if "/" in candidate and candidate.lower() != "unknown filetype":
mime_type = candidate
if not mime_type:
mime_type = _mime_from_ext(ext)
# Normalize size/duration to stable scalar types.
size_val = meta.get("size")
if size_val is None:
size_val = meta.get("size_bytes")
try:
size_int: int | None = int(size_val) if size_val is not None else None
except Exception:
size_int = None
dur_val = meta.get("duration")
if dur_val is None:
dur_val = meta.get("duration_ms")
try:
dur_int: int | None = int(dur_val) if dur_val is not None else None
except Exception:
dur_int = None
raw_urls = (
meta.get("known_urls")
or meta.get("urls")
or meta.get("url")
or []
)
url_list: list[str] = []
if isinstance(raw_urls, str):
s = raw_urls.strip()
url_list = [s] if s else []
elif isinstance(raw_urls, list):
url_list = [str(u).strip() for u in raw_urls if isinstance(u, str) and str(u).strip()]
return {
"hash": file_hash,
"title": title,
"ext": ext,
"size": meta.get("size"),
"size": size_int,
"mime": mime_type,
# Keep raw fields available for troubleshooting/other callers.
"hydrus_mime": mime_val,
"filetype_human": filetype_human,
"duration_ms": dur_int,
"url": url_list,
}
except Exception as exc:
debug(f"Failed to get metadata from Hydrus: {exc}")
debug(f"{self._log_prefix()} get_metadata failed: {exc}")
return None
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
@@ -705,13 +818,13 @@ class HydrusNetwork(Store):
file_hash = str(file_identifier or "").strip().lower()
if len(file_hash) != 64 or not all(ch in "0123456789abcdef" for ch in file_hash):
debug(f"get_tags: invalid file hash '{file_identifier}'")
debug(f"{self._log_prefix()} get_tags: invalid file hash '{file_identifier}'")
return [], "unknown"
# Get Hydrus client and service info
client = self._client
if not client:
debug("get_tags: Hydrus client unavailable")
debug(f"{self._log_prefix()} get_tags: client unavailable")
return [], "unknown"
# Fetch file metadata
@@ -723,12 +836,12 @@ class HydrusNetwork(Store):
items = payload.get("metadata") if isinstance(payload, dict) else None
if not isinstance(items, list) or not items:
debug(f"get_tags: No metadata returned for hash {file_hash}")
debug(f"{self._log_prefix()} get_tags: no metadata for hash {file_hash}")
return [], "unknown"
meta = items[0] if isinstance(items[0], dict) else None
if not isinstance(meta, dict) or meta.get("file_id") is None:
debug(f"get_tags: Invalid metadata for hash {file_hash}")
debug(f"{self._log_prefix()} get_tags: invalid metadata for hash {file_hash}")
return [], "unknown"
# Extract tags using service name
@@ -741,7 +854,7 @@ class HydrusNetwork(Store):
return tags, "hydrus"
except Exception as exc:
debug(f"get_tags failed for Hydrus file: {exc}")
debug(f"{self._log_prefix()} get_tags failed: {exc}")
return [], "unknown"
def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
@@ -750,12 +863,12 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("add_tag: Hydrus client unavailable")
debug(f"{self._log_prefix()} add_tag: client unavailable")
return False
file_hash = str(file_identifier or "").strip().lower()
if len(file_hash) != 64 or not all(ch in "0123456789abcdef" for ch in file_hash):
debug(f"add_tag: invalid file hash '{file_identifier}'")
debug(f"{self._log_prefix()} add_tag: invalid file hash '{file_identifier}'")
return False
service_name = kwargs.get("service_name") or "my tags"
# Ensure tags is a list
@@ -765,7 +878,7 @@ class HydrusNetwork(Store):
client.add_tag(file_hash, tag_list, service_name)
return True
except Exception as exc:
debug(f"Hydrus add_tag failed: {exc}")
debug(f"{self._log_prefix()} add_tag failed: {exc}")
return False
def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
@@ -774,12 +887,12 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("delete_tag: Hydrus client unavailable")
debug(f"{self._log_prefix()} delete_tag: client unavailable")
return False
file_hash = str(file_identifier or "").strip().lower()
if len(file_hash) != 64 or not all(ch in "0123456789abcdef" for ch in file_hash):
debug(f"delete_tag: invalid file hash '{file_identifier}'")
debug(f"{self._log_prefix()} delete_tag: invalid file hash '{file_identifier}'")
return False
service_name = kwargs.get("service_name") or "my tags"
tag_list = list(tags) if isinstance(tags, (list, tuple)) else [str(tags)]
@@ -788,7 +901,7 @@ class HydrusNetwork(Store):
client.delete_tag(file_hash, tag_list, service_name)
return True
except Exception as exc:
debug(f"Hydrus delete_tag failed: {exc}")
debug(f"{self._log_prefix()} delete_tag failed: {exc}")
return False
def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]:
@@ -797,7 +910,7 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("get_url: Hydrus client unavailable")
debug(f"{self._log_prefix()} get_url: client unavailable")
return []
file_hash = str(file_identifier or "").strip().lower()
@@ -830,7 +943,7 @@ class HydrusNetwork(Store):
return out
return []
except Exception as exc:
debug(f"Hydrus get_url failed: {exc}")
debug(f"{self._log_prefix()} get_url failed: {exc}")
return []
def add_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool:
@@ -839,13 +952,13 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("add_url: Hydrus client unavailable")
debug(f"{self._log_prefix()} add_url: client unavailable")
return False
for u in url:
client.associate_url(file_identifier, u)
return True
except Exception as exc:
debug(f"Hydrus add_url failed: {exc}")
debug(f"{self._log_prefix()} add_url failed: {exc}")
return False
def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool:
@@ -854,13 +967,13 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("delete_url: Hydrus client unavailable")
debug(f"{self._log_prefix()} delete_url: client unavailable")
return False
for u in url:
client.delete_url(file_identifier, u)
return True
except Exception as exc:
debug(f"Hydrus delete_url failed: {exc}")
debug(f"{self._log_prefix()} delete_url failed: {exc}")
return False
def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]:
@@ -868,7 +981,7 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("get_note: Hydrus client unavailable")
debug(f"{self._log_prefix()} get_note: client unavailable")
return {}
file_hash = str(file_identifier or "").strip().lower()
@@ -889,7 +1002,7 @@ class HydrusNetwork(Store):
return {}
except Exception as exc:
debug(f"Hydrus get_note failed: {exc}")
debug(f"{self._log_prefix()} get_note failed: {exc}")
return {}
def set_note(self, file_identifier: str, name: str, text: str, **kwargs: Any) -> bool:
@@ -897,7 +1010,7 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("set_note: Hydrus client unavailable")
debug(f"{self._log_prefix()} set_note: client unavailable")
return False
file_hash = str(file_identifier or "").strip().lower()
@@ -912,7 +1025,7 @@ class HydrusNetwork(Store):
client.set_notes(file_hash, {note_name: note_text})
return True
except Exception as exc:
debug(f"Hydrus set_note failed: {exc}")
debug(f"{self._log_prefix()} set_note failed: {exc}")
return False
def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool:
@@ -920,7 +1033,7 @@ class HydrusNetwork(Store):
try:
client = self._client
if client is None:
debug("delete_note: Hydrus client unavailable")
debug(f"{self._log_prefix()} delete_note: client unavailable")
return False
file_hash = str(file_identifier or "").strip().lower()
@@ -934,7 +1047,7 @@ class HydrusNetwork(Store):
client.delete_notes(file_hash, [note_name])
return True
except Exception as exc:
debug(f"Hydrus delete_note failed: {exc}")
debug(f"{self._log_prefix()} delete_note failed: {exc}")
return False
@staticmethod