df
Some checks failed
smoke-mm / Install & smoke test mm --help (push) Has been cancelled
Some checks failed
smoke-mm / Install & smoke test mm --help (push) Has been cancelled
This commit is contained in:
514
Store/Folder.py
514
Store/Folder.py
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,7 @@ _HYDRUS_INIT_CHECK_CACHE: dict[tuple[str, str], tuple[bool, Optional[str]]] = {}
|
||||
|
||||
class HydrusNetwork(Store):
|
||||
"""File storage backend for Hydrus client.
|
||||
|
||||
|
||||
Each instance represents a specific Hydrus client connection.
|
||||
Maintains its own HydrusClient.
|
||||
"""
|
||||
@@ -41,7 +41,7 @@ class HydrusNetwork(Store):
|
||||
return instance
|
||||
|
||||
setattr(__new__, "keys", ("NAME", "API", "URL"))
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_name: Optional[str] = None,
|
||||
@@ -53,7 +53,7 @@ class HydrusNetwork(Store):
|
||||
URL: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Initialize Hydrus storage backend.
|
||||
|
||||
|
||||
Args:
|
||||
instance_name: Name of this Hydrus instance (e.g., 'home', 'work')
|
||||
api_key: Hydrus Client API access key
|
||||
@@ -70,7 +70,7 @@ class HydrusNetwork(Store):
|
||||
|
||||
if not instance_name or not api_key or not url:
|
||||
raise ValueError("HydrusNetwork requires NAME, API, and URL")
|
||||
|
||||
|
||||
self.NAME = instance_name
|
||||
self.API = api_key
|
||||
self.URL = url.rstrip("/")
|
||||
@@ -104,7 +104,9 @@ class HydrusNetwork(Store):
|
||||
verify_resp.raise_for_status()
|
||||
verify_payload = verify_resp.json()
|
||||
if not isinstance(verify_payload, dict):
|
||||
raise RuntimeError("Hydrus /verify_access_key returned an unexpected response")
|
||||
raise RuntimeError(
|
||||
"Hydrus /verify_access_key returned an unexpected response"
|
||||
)
|
||||
|
||||
_HYDRUS_INIT_CHECK_CACHE[cache_key] = (True, None)
|
||||
except Exception as exc:
|
||||
@@ -196,16 +198,16 @@ class HydrusNetwork(Store):
|
||||
|
||||
def add_file(self, file_path: Path, **kwargs: Any) -> str:
|
||||
"""Upload file to Hydrus with full metadata support.
|
||||
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
tag: Optional list of tag values to add
|
||||
url: Optional list of url to associate with the file
|
||||
title: Optional title (will be added as 'title:value' tag)
|
||||
|
||||
|
||||
Returns:
|
||||
File hash from Hydrus
|
||||
|
||||
|
||||
Raises:
|
||||
Exception: If upload fails
|
||||
"""
|
||||
@@ -214,7 +216,7 @@ class HydrusNetwork(Store):
|
||||
tag_list = kwargs.get("tag", [])
|
||||
url = kwargs.get("url", [])
|
||||
title = kwargs.get("title")
|
||||
|
||||
|
||||
# Add title to tags if provided and not already present
|
||||
if title:
|
||||
title_tag = f"title:{title}".strip().lower()
|
||||
@@ -222,7 +224,11 @@ class HydrusNetwork(Store):
|
||||
tag_list = [title_tag] + list(tag_list)
|
||||
|
||||
# Hydrus is lowercase-only tags; normalize here for consistency.
|
||||
tag_list = [str(t).strip().lower() for t in (tag_list or []) if isinstance(t, str) and str(t).strip()]
|
||||
tag_list = [
|
||||
str(t).strip().lower()
|
||||
for t in (tag_list or [])
|
||||
if isinstance(t, str) and str(t).strip()
|
||||
]
|
||||
|
||||
try:
|
||||
# Compute file hash
|
||||
@@ -307,14 +313,19 @@ class HydrusNetwork(Store):
|
||||
|
||||
# Associate url if provided (both for new and existing files)
|
||||
if url:
|
||||
log(f"{self._log_prefix()} Associating {len(url)} URL(s) with file", file=sys.stderr)
|
||||
log(
|
||||
f"{self._log_prefix()} Associating {len(url)} URL(s) with file", file=sys.stderr
|
||||
)
|
||||
for url in url:
|
||||
if url:
|
||||
try:
|
||||
client.associate_url(file_hash, str(url))
|
||||
debug(f"{self._log_prefix()} Associated URL: {url}")
|
||||
except Exception as exc:
|
||||
log(f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}", file=sys.stderr)
|
||||
log(
|
||||
f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
return file_hash
|
||||
|
||||
@@ -324,14 +335,14 @@ class HydrusNetwork(Store):
|
||||
|
||||
def search(self, query: str, **kwargs: Any) -> list[Dict[str, Any]]:
|
||||
"""Search Hydrus database for files matching query.
|
||||
|
||||
|
||||
Args:
|
||||
query: Search query (tags, filenames, hashes, etc.)
|
||||
limit: Maximum number of results to return (default: 100)
|
||||
|
||||
|
||||
Returns:
|
||||
List of dicts with 'name', 'hash', 'size', 'tags' fields
|
||||
|
||||
|
||||
Example:
|
||||
results = storage["hydrus"].search("artist:john_doe music")
|
||||
results = storage["hydrus"].search("Simple Man")
|
||||
@@ -366,7 +377,9 @@ class HydrusNetwork(Store):
|
||||
return out
|
||||
return []
|
||||
|
||||
def _iter_url_filtered_metadata(url_value: str | None, want_any: bool, fetch_limit: int) -> list[dict[str, Any]]:
|
||||
def _iter_url_filtered_metadata(
|
||||
url_value: str | None, want_any: bool, fetch_limit: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Best-effort URL search by scanning Hydrus metadata with include_file_url=True."""
|
||||
|
||||
# First try a fast system predicate if Hydrus supports it.
|
||||
@@ -382,7 +395,11 @@ class HydrusNetwork(Store):
|
||||
)
|
||||
ids = url_search.get("file_ids", []) if isinstance(url_search, dict) else []
|
||||
if isinstance(ids, list):
|
||||
candidate_file_ids = [int(x) for x in ids if isinstance(x, (int, float, str)) and str(x).strip().isdigit()]
|
||||
candidate_file_ids = [
|
||||
int(x)
|
||||
for x in ids
|
||||
if isinstance(x, (int, float, str)) and str(x).strip().isdigit()
|
||||
]
|
||||
except Exception:
|
||||
candidate_file_ids = []
|
||||
|
||||
@@ -451,7 +468,7 @@ class HydrusNetwork(Store):
|
||||
# Support `ext:<value>` anywhere in the query. We filter results by the
|
||||
# Hydrus metadata extension field.
|
||||
def _normalize_ext_filter(value: str) -> str:
|
||||
v = str(value or "").strip().lower().lstrip('.')
|
||||
v = str(value or "").strip().lower().lstrip(".")
|
||||
v = "".join(ch for ch in v if ch.isalnum())
|
||||
return v
|
||||
|
||||
@@ -464,7 +481,7 @@ class HydrusNetwork(Store):
|
||||
if m:
|
||||
ext_filter = _normalize_ext_filter(m.group(1)) or None
|
||||
query_lower = re.sub(r"\s*\b(?:ext|extension):[^\s,]+", " ", query_lower)
|
||||
query_lower = re.sub(r"\s{2,}", " ", query_lower).strip().strip(',')
|
||||
query_lower = re.sub(r"\s{2,}", " ", query_lower).strip().strip(",")
|
||||
query = query_lower
|
||||
if ext_filter and not query_lower:
|
||||
query = "*"
|
||||
@@ -486,21 +503,33 @@ class HydrusNetwork(Store):
|
||||
pattern = pattern.strip()
|
||||
if namespace == "url":
|
||||
if not pattern or pattern == "*":
|
||||
metadata_list = _iter_url_filtered_metadata(None, want_any=True, fetch_limit=int(limit) if limit else 100)
|
||||
metadata_list = _iter_url_filtered_metadata(
|
||||
None, want_any=True, fetch_limit=int(limit) if limit else 100
|
||||
)
|
||||
else:
|
||||
# Fast-path: exact URL via /add_urls/get_url_files when a full URL is provided.
|
||||
try:
|
||||
if pattern.startswith("http://") or pattern.startswith("https://"):
|
||||
from API.HydrusNetwork import HydrusRequestSpec
|
||||
|
||||
spec = HydrusRequestSpec(method="GET", endpoint="/add_urls/get_url_files", query={"url": pattern})
|
||||
spec = HydrusRequestSpec(
|
||||
method="GET",
|
||||
endpoint="/add_urls/get_url_files",
|
||||
query={"url": pattern},
|
||||
)
|
||||
response = client._perform_request(spec) # type: ignore[attr-defined]
|
||||
hashes: list[str] = []
|
||||
file_ids: list[int] = []
|
||||
if isinstance(response, dict):
|
||||
raw_hashes = response.get("hashes") or response.get("file_hashes")
|
||||
raw_hashes = response.get("hashes") or response.get(
|
||||
"file_hashes"
|
||||
)
|
||||
if isinstance(raw_hashes, list):
|
||||
hashes = [str(h).strip() for h in raw_hashes if isinstance(h, str) and str(h).strip()]
|
||||
hashes = [
|
||||
str(h).strip()
|
||||
for h in raw_hashes
|
||||
if isinstance(h, str) and str(h).strip()
|
||||
]
|
||||
raw_ids = response.get("file_ids")
|
||||
if isinstance(raw_ids, list):
|
||||
for item in raw_ids:
|
||||
@@ -518,7 +547,11 @@ class HydrusNetwork(Store):
|
||||
include_size=True,
|
||||
include_mime=True,
|
||||
)
|
||||
metas = payload.get("metadata", []) if isinstance(payload, dict) else []
|
||||
metas = (
|
||||
payload.get("metadata", [])
|
||||
if isinstance(payload, dict)
|
||||
else []
|
||||
)
|
||||
if isinstance(metas, list):
|
||||
metadata_list = [m for m in metas if isinstance(m, dict)]
|
||||
elif hashes:
|
||||
@@ -530,7 +563,11 @@ class HydrusNetwork(Store):
|
||||
include_size=True,
|
||||
include_mime=True,
|
||||
)
|
||||
metas = payload.get("metadata", []) if isinstance(payload, dict) else []
|
||||
metas = (
|
||||
payload.get("metadata", [])
|
||||
if isinstance(payload, dict)
|
||||
else []
|
||||
)
|
||||
if isinstance(metas, list):
|
||||
metadata_list = [m for m in metas if isinstance(m, dict)]
|
||||
except Exception:
|
||||
@@ -538,7 +575,9 @@ class HydrusNetwork(Store):
|
||||
|
||||
# Fallback: substring scan
|
||||
if metadata_list is None:
|
||||
metadata_list = _iter_url_filtered_metadata(pattern, want_any=False, fetch_limit=int(limit) if limit else 100)
|
||||
metadata_list = _iter_url_filtered_metadata(
|
||||
pattern, want_any=False, fetch_limit=int(limit) if limit else 100
|
||||
)
|
||||
|
||||
# Parse the query into tags
|
||||
# "*" means "match all" - use system:everything tag in Hydrus
|
||||
@@ -553,7 +592,7 @@ class HydrusNetwork(Store):
|
||||
|
||||
if query.strip() == "*":
|
||||
tags = ["system:everything"]
|
||||
elif ':' in query_lower:
|
||||
elif ":" in query_lower:
|
||||
tags = [query_lower]
|
||||
else:
|
||||
freeform_union_search = True
|
||||
@@ -566,7 +605,7 @@ class HydrusNetwork(Store):
|
||||
# If we can't extract alnum terms, fall back to the raw query text.
|
||||
title_predicates = [f"title:{query_lower}*"]
|
||||
freeform_predicates = [f"{query_lower}*"]
|
||||
|
||||
|
||||
# Search files with the tags (unless url: search already produced metadata)
|
||||
results = []
|
||||
|
||||
@@ -584,7 +623,9 @@ class HydrusNetwork(Store):
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if isinstance(raw_hashes, list):
|
||||
hashes_out = [str(h).strip() for h in raw_hashes if isinstance(h, str) and str(h).strip()]
|
||||
hashes_out = [
|
||||
str(h).strip() for h in raw_hashes if isinstance(h, str) and str(h).strip()
|
||||
]
|
||||
return ids_out, hashes_out
|
||||
|
||||
if metadata_list is None:
|
||||
@@ -635,9 +676,7 @@ class HydrusNetwork(Store):
|
||||
return []
|
||||
|
||||
search_result = client.search_files(
|
||||
tags=tags,
|
||||
return_hashes=True,
|
||||
return_file_ids=True
|
||||
tags=tags, return_hashes=True, return_file_ids=True
|
||||
)
|
||||
file_ids, hashes = _extract_search_ids(search_result)
|
||||
|
||||
@@ -676,12 +715,12 @@ class HydrusNetwork(Store):
|
||||
if not isinstance(meta, dict):
|
||||
continue
|
||||
mime_type = meta.get("mime")
|
||||
ext = str(meta.get("ext") or "").strip().lstrip('.')
|
||||
ext = str(meta.get("ext") or "").strip().lstrip(".")
|
||||
if not ext and mime_type:
|
||||
for category in mime_maps.values():
|
||||
for _ext_key, info in category.items():
|
||||
if mime_type in info.get("mimes", []):
|
||||
ext = str(info.get("ext", "")).strip().lstrip('.')
|
||||
ext = str(info.get("ext", "")).strip().lstrip(".")
|
||||
break
|
||||
if ext:
|
||||
break
|
||||
@@ -696,6 +735,7 @@ class HydrusNetwork(Store):
|
||||
all_tags: list[str] = []
|
||||
title = f"Hydrus File {file_id}"
|
||||
if isinstance(tags_set, dict):
|
||||
|
||||
def _collect(tag_list: Any) -> None:
|
||||
nonlocal title
|
||||
if not isinstance(tag_list, list):
|
||||
@@ -708,7 +748,10 @@ class HydrusNetwork(Store):
|
||||
if not tag_l:
|
||||
continue
|
||||
all_tags.append(tag_l)
|
||||
if tag_l.startswith("title:") and title == f"Hydrus File {file_id}":
|
||||
if (
|
||||
tag_l.startswith("title:")
|
||||
and title == f"Hydrus File {file_id}"
|
||||
):
|
||||
title = tag_l.split(":", 1)[1].strip()
|
||||
|
||||
for _service_name, service_tags in tags_set.items():
|
||||
@@ -807,77 +850,78 @@ class HydrusNetwork(Store):
|
||||
metadata_list = []
|
||||
|
||||
for meta in metadata_list:
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
file_id = meta.get("file_id")
|
||||
hash_hex = meta.get("hash")
|
||||
size = meta.get("size", 0)
|
||||
|
||||
# Get tags for this file and extract title
|
||||
tags_set = meta.get("tags", {})
|
||||
all_tags = []
|
||||
title = f"Hydrus File {file_id}" # Default fallback
|
||||
all_tags_str = "" # For substring matching
|
||||
|
||||
# debug(f"[HydrusBackend.search] Processing file_id={file_id}, tags type={type(tags_set)}")
|
||||
|
||||
if isinstance(tags_set, dict):
|
||||
# Collect both storage_tags and display_tags to capture siblings/parents and ensure title: is seen
|
||||
def _collect(tag_list: Any) -> None:
|
||||
nonlocal title, all_tags_str
|
||||
if not isinstance(tag_list, list):
|
||||
return
|
||||
for tag in tag_list:
|
||||
tag_text = str(tag) if tag else ""
|
||||
if not tag_text:
|
||||
continue
|
||||
tag_l = tag_text.strip().lower()
|
||||
if not tag_l:
|
||||
continue
|
||||
all_tags.append(tag_l)
|
||||
all_tags_str += " " + tag_l
|
||||
if tag_l.startswith("title:") and title == f"Hydrus File {file_id}":
|
||||
title = tag_l.split(":", 1)[1].strip()
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
for _service_name, service_tags in tags_set.items():
|
||||
if not isinstance(service_tags, dict):
|
||||
file_id = meta.get("file_id")
|
||||
hash_hex = meta.get("hash")
|
||||
size = meta.get("size", 0)
|
||||
|
||||
# Get tags for this file and extract title
|
||||
tags_set = meta.get("tags", {})
|
||||
all_tags = []
|
||||
title = f"Hydrus File {file_id}" # Default fallback
|
||||
all_tags_str = "" # For substring matching
|
||||
|
||||
# debug(f"[HydrusBackend.search] Processing file_id={file_id}, tags type={type(tags_set)}")
|
||||
|
||||
if isinstance(tags_set, dict):
|
||||
# Collect both storage_tags and display_tags to capture siblings/parents and ensure title: is seen
|
||||
def _collect(tag_list: Any) -> None:
|
||||
nonlocal title, all_tags_str
|
||||
if not isinstance(tag_list, list):
|
||||
return
|
||||
for tag in tag_list:
|
||||
tag_text = str(tag) if tag else ""
|
||||
if not tag_text:
|
||||
continue
|
||||
tag_l = tag_text.strip().lower()
|
||||
if not tag_l:
|
||||
continue
|
||||
all_tags.append(tag_l)
|
||||
all_tags_str += " " + tag_l
|
||||
if tag_l.startswith("title:") and title == f"Hydrus File {file_id}":
|
||||
title = tag_l.split(":", 1)[1].strip()
|
||||
|
||||
storage_tags = service_tags.get("storage_tags", {})
|
||||
if isinstance(storage_tags, dict):
|
||||
for tag_list in storage_tags.values():
|
||||
_collect(tag_list)
|
||||
for _service_name, service_tags in tags_set.items():
|
||||
if not isinstance(service_tags, dict):
|
||||
continue
|
||||
|
||||
display_tags = service_tags.get("display_tags", [])
|
||||
_collect(display_tags)
|
||||
storage_tags = service_tags.get("storage_tags", {})
|
||||
if isinstance(storage_tags, dict):
|
||||
for tag_list in storage_tags.values():
|
||||
_collect(tag_list)
|
||||
|
||||
# Also consider top-level flattened tags payload if provided (Hydrus API sometimes includes it)
|
||||
top_level_tags = meta.get("tags_flat", []) or meta.get("tags", [])
|
||||
_collect(top_level_tags)
|
||||
|
||||
# Prefer Hydrus-provided extension (e.g. ".webm"); fall back to MIME map.
|
||||
mime_type = meta.get("mime")
|
||||
ext = str(meta.get("ext") or "").strip().lstrip('.')
|
||||
if not ext and mime_type:
|
||||
for category in mime_maps.values():
|
||||
for _ext_key, info in category.items():
|
||||
if mime_type in info.get("mimes", []):
|
||||
ext = str(info.get("ext", "")).strip().lstrip('.')
|
||||
break
|
||||
if ext:
|
||||
display_tags = service_tags.get("display_tags", [])
|
||||
_collect(display_tags)
|
||||
|
||||
# Also consider top-level flattened tags payload if provided (Hydrus API sometimes includes it)
|
||||
top_level_tags = meta.get("tags_flat", []) or meta.get("tags", [])
|
||||
_collect(top_level_tags)
|
||||
|
||||
# Prefer Hydrus-provided extension (e.g. ".webm"); fall back to MIME map.
|
||||
mime_type = meta.get("mime")
|
||||
ext = str(meta.get("ext") or "").strip().lstrip(".")
|
||||
if not ext and mime_type:
|
||||
for category in mime_maps.values():
|
||||
for _ext_key, info in category.items():
|
||||
if mime_type in info.get("mimes", []):
|
||||
ext = str(info.get("ext", "")).strip().lstrip(".")
|
||||
break
|
||||
if ext:
|
||||
break
|
||||
|
||||
# Filter results based on query type
|
||||
# If user provided explicit namespace (has ':'), don't do substring filtering
|
||||
# Just include what the tag search returned
|
||||
has_namespace = ':' in query_lower
|
||||
|
||||
if has_namespace:
|
||||
# Explicit namespace search - already filtered by Hydrus tag search
|
||||
# Include this result as-is
|
||||
file_url = f"{self.URL.rstrip('/')}/get_files/file?hash={hash_hex}"
|
||||
results.append({
|
||||
# Filter results based on query type
|
||||
# If user provided explicit namespace (has ':'), don't do substring filtering
|
||||
# Just include what the tag search returned
|
||||
has_namespace = ":" in query_lower
|
||||
|
||||
if has_namespace:
|
||||
# Explicit namespace search - already filtered by Hydrus tag search
|
||||
# Include this result as-is
|
||||
file_url = f"{self.URL.rstrip('/')}/get_files/file?hash={hash_hex}"
|
||||
results.append(
|
||||
{
|
||||
"hash": hash_hex,
|
||||
"url": file_url,
|
||||
"name": title,
|
||||
@@ -885,27 +929,31 @@ class HydrusNetwork(Store):
|
||||
"size": size,
|
||||
"size_bytes": size,
|
||||
"store": self.NAME,
|
||||
"tag": all_tags,
|
||||
"tag": all_tags,
|
||||
"file_id": file_id,
|
||||
"mime": mime_type,
|
||||
"ext": ext,
|
||||
})
|
||||
else:
|
||||
# Free-form search: check if search terms match title or FREEFORM tags.
|
||||
# Do NOT implicitly match other namespace tags (except title:).
|
||||
freeform_tags = [t for t in all_tags if isinstance(t, str) and t and (":" not in t)]
|
||||
searchable_text = (title + " " + " ".join(freeform_tags)).lower()
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Free-form search: check if search terms match title or FREEFORM tags.
|
||||
# Do NOT implicitly match other namespace tags (except title:).
|
||||
freeform_tags = [
|
||||
t for t in all_tags if isinstance(t, str) and t and (":" not in t)
|
||||
]
|
||||
searchable_text = (title + " " + " ".join(freeform_tags)).lower()
|
||||
|
||||
match = True
|
||||
if query_lower != "*" and search_terms:
|
||||
for term in search_terms:
|
||||
if term not in searchable_text:
|
||||
match = False
|
||||
break
|
||||
|
||||
if match:
|
||||
file_url = f"{self.URL.rstrip('/')}/get_files/file?hash={hash_hex}"
|
||||
results.append({
|
||||
match = True
|
||||
if query_lower != "*" and search_terms:
|
||||
for term in search_terms:
|
||||
if term not in searchable_text:
|
||||
match = False
|
||||
break
|
||||
|
||||
if match:
|
||||
file_url = f"{self.URL.rstrip('/')}/get_files/file?hash={hash_hex}"
|
||||
results.append(
|
||||
{
|
||||
"hash": hash_hex,
|
||||
"url": file_url,
|
||||
"name": title,
|
||||
@@ -917,8 +965,9 @@ class HydrusNetwork(Store):
|
||||
"file_id": file_id,
|
||||
"mime": mime_type,
|
||||
"ext": ext,
|
||||
})
|
||||
|
||||
}
|
||||
)
|
||||
|
||||
debug(f"{prefix} {len(results)} result(s)")
|
||||
if ext_filter:
|
||||
wanted = ext_filter
|
||||
@@ -936,6 +985,7 @@ class HydrusNetwork(Store):
|
||||
except Exception as exc:
|
||||
log(f"❌ Hydrus search failed: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
raise
|
||||
|
||||
@@ -945,13 +995,15 @@ class HydrusNetwork(Store):
|
||||
IMPORTANT: this method must be side-effect free (do not auto-open a browser).
|
||||
Only explicit user actions (e.g. the get-file cmdlet) should open files.
|
||||
"""
|
||||
|
||||
|
||||
debug(f"{self._log_prefix()} get_file: start hash={file_hash[:12]}...")
|
||||
|
||||
|
||||
# Build browser URL with access key
|
||||
base_url = str(self.URL).rstrip('/')
|
||||
base_url = str(self.URL).rstrip("/")
|
||||
access_key = str(self.API)
|
||||
browser_url = f"{base_url}/get_files/file?hash={file_hash}&Hydrus-Client-API-Access-Key={access_key}"
|
||||
browser_url = (
|
||||
f"{base_url}/get_files/file?hash={file_hash}&Hydrus-Client-API-Access-Key={access_key}"
|
||||
)
|
||||
debug(f"{self._log_prefix()} get_file: url={browser_url}")
|
||||
return browser_url
|
||||
|
||||
@@ -972,7 +1024,9 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
|
||||
reason = kwargs.get("reason")
|
||||
reason_text = str(reason).strip() if isinstance(reason, str) and reason.strip() else None
|
||||
reason_text = (
|
||||
str(reason).strip() if isinstance(reason, str) and reason.strip() else None
|
||||
)
|
||||
|
||||
# 1) Delete file
|
||||
client.delete_files([file_hash], reason=reason_text)
|
||||
@@ -990,10 +1044,10 @@ class HydrusNetwork(Store):
|
||||
|
||||
def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
|
||||
"""Get metadata for a file from Hydrus by hash.
|
||||
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of the file (64-char hex string)
|
||||
|
||||
|
||||
Returns:
|
||||
Dict with metadata fields or None if not found
|
||||
"""
|
||||
@@ -1002,7 +1056,7 @@ class HydrusNetwork(Store):
|
||||
if not client:
|
||||
debug(f"{self._log_prefix()} get_metadata: client unavailable")
|
||||
return None
|
||||
|
||||
|
||||
# Fetch file metadata with the fields we need for CLI display.
|
||||
payload = client.fetch_file_metadata(
|
||||
hashes=[file_hash],
|
||||
@@ -1012,16 +1066,16 @@ class HydrusNetwork(Store):
|
||||
include_size=True,
|
||||
include_mime=True,
|
||||
)
|
||||
|
||||
|
||||
if not payload or not payload.get("metadata"):
|
||||
return None
|
||||
|
||||
|
||||
meta = payload["metadata"][0]
|
||||
|
||||
# Hydrus can return placeholder metadata rows for unknown hashes.
|
||||
if not isinstance(meta, dict) or meta.get("file_id") is None:
|
||||
return None
|
||||
|
||||
|
||||
# Extract title from tags
|
||||
title = f"Hydrus_{file_hash[:12]}"
|
||||
tags_payload = meta.get("tags", {})
|
||||
@@ -1038,10 +1092,12 @@ class HydrusNetwork(Store):
|
||||
break
|
||||
if title != f"Hydrus_{file_hash[:12]}":
|
||||
break
|
||||
|
||||
|
||||
# Hydrus may return mime as an int enum, or sometimes a human label.
|
||||
mime_val = meta.get("mime")
|
||||
filetype_human = meta.get("filetype_human") or meta.get("mime_human") or meta.get("mime_string")
|
||||
filetype_human = (
|
||||
meta.get("filetype_human") or meta.get("mime_human") or meta.get("mime_string")
|
||||
)
|
||||
|
||||
# Determine ext: prefer Hydrus metadata ext, then filetype_human (when it looks like an ext),
|
||||
# then title suffix, then file path suffix.
|
||||
@@ -1113,19 +1169,16 @@ class HydrusNetwork(Store):
|
||||
dur_int: int | None = int(dur_val) if dur_val is not None else None
|
||||
except Exception:
|
||||
dur_int = None
|
||||
|
||||
raw_urls = (
|
||||
meta.get("known_urls")
|
||||
or meta.get("urls")
|
||||
or meta.get("url")
|
||||
or []
|
||||
)
|
||||
|
||||
raw_urls = meta.get("known_urls") or meta.get("urls") or meta.get("url") or []
|
||||
url_list: list[str] = []
|
||||
if isinstance(raw_urls, str):
|
||||
s = raw_urls.strip()
|
||||
url_list = [s] if s else []
|
||||
elif isinstance(raw_urls, list):
|
||||
url_list = [str(u).strip() for u in raw_urls if isinstance(u, str) and str(u).strip()]
|
||||
url_list = [
|
||||
str(u).strip() for u in raw_urls if isinstance(u, str) and str(u).strip()
|
||||
]
|
||||
|
||||
return {
|
||||
"hash": file_hash,
|
||||
@@ -1139,18 +1192,18 @@ class HydrusNetwork(Store):
|
||||
"duration_ms": dur_int,
|
||||
"url": url_list,
|
||||
}
|
||||
|
||||
|
||||
except Exception as exc:
|
||||
debug(f"{self._log_prefix()} get_metadata failed: {exc}")
|
||||
return None
|
||||
|
||||
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
|
||||
"""Get tags for a file from Hydrus by hash.
|
||||
|
||||
|
||||
Args:
|
||||
file_identifier: File hash (SHA256 hex string)
|
||||
**kwargs: Optional service_name parameter
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (tags_list, source_description)
|
||||
where source is always "hydrus"
|
||||
@@ -1162,46 +1215,45 @@ class HydrusNetwork(Store):
|
||||
if len(file_hash) != 64 or not all(ch in "0123456789abcdef" for ch in file_hash):
|
||||
debug(f"{self._log_prefix()} get_tags: invalid file hash '{file_identifier}'")
|
||||
return [], "unknown"
|
||||
|
||||
|
||||
# Get Hydrus client and service info
|
||||
client = self._client
|
||||
if not client:
|
||||
debug(f"{self._log_prefix()} get_tags: client unavailable")
|
||||
return [], "unknown"
|
||||
|
||||
|
||||
# Fetch file metadata
|
||||
payload = client.fetch_file_metadata(
|
||||
hashes=[file_hash],
|
||||
include_service_keys_to_tags=True,
|
||||
include_file_url=False
|
||||
hashes=[file_hash], include_service_keys_to_tags=True, include_file_url=False
|
||||
)
|
||||
|
||||
|
||||
items = payload.get("metadata") if isinstance(payload, dict) else None
|
||||
if not isinstance(items, list) or not items:
|
||||
debug(f"{self._log_prefix()} get_tags: no metadata for hash {file_hash}")
|
||||
return [], "unknown"
|
||||
|
||||
|
||||
meta = items[0] if isinstance(items[0], dict) else None
|
||||
if not isinstance(meta, dict) or meta.get("file_id") is None:
|
||||
debug(f"{self._log_prefix()} get_tags: invalid metadata for hash {file_hash}")
|
||||
return [], "unknown"
|
||||
|
||||
|
||||
# Extract tags using service name
|
||||
service_name = "my tags"
|
||||
service_key = hydrus_wrapper.get_tag_service_key(client, service_name)
|
||||
|
||||
|
||||
# Extract tags from metadata
|
||||
tags = self._extract_tags_from_hydrus_meta(meta, service_key, service_name)
|
||||
|
||||
return [str(t).strip().lower() for t in tags if isinstance(t, str) and t.strip()], "hydrus"
|
||||
|
||||
return [
|
||||
str(t).strip().lower() for t in tags if isinstance(t, str) and t.strip()
|
||||
], "hydrus"
|
||||
|
||||
except Exception as exc:
|
||||
debug(f"{self._log_prefix()} get_tags failed: {exc}")
|
||||
return [], "unknown"
|
||||
|
||||
def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
|
||||
"""Add tags to a Hydrus file.
|
||||
"""
|
||||
"""Add tags to a Hydrus file."""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
@@ -1214,7 +1266,11 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
service_name = kwargs.get("service_name") or "my tags"
|
||||
|
||||
incoming_tags = [str(t).strip().lower() for t in (tags or []) if isinstance(t, str) and str(t).strip()]
|
||||
incoming_tags = [
|
||||
str(t).strip().lower()
|
||||
for t in (tags or [])
|
||||
if isinstance(t, str) and str(t).strip()
|
||||
]
|
||||
if not incoming_tags:
|
||||
return True
|
||||
|
||||
@@ -1225,7 +1281,9 @@ class HydrusNetwork(Store):
|
||||
|
||||
from metadata import compute_namespaced_tag_overwrite
|
||||
|
||||
tags_to_remove, tags_to_add, _merged = compute_namespaced_tag_overwrite(existing_tags, incoming_tags)
|
||||
tags_to_remove, tags_to_add, _merged = compute_namespaced_tag_overwrite(
|
||||
existing_tags, incoming_tags
|
||||
)
|
||||
|
||||
if not tags_to_add and not tags_to_remove:
|
||||
return True
|
||||
@@ -1250,8 +1308,7 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
|
||||
def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
|
||||
"""Delete tags from a Hydrus file.
|
||||
"""
|
||||
"""Delete tags from a Hydrus file."""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
@@ -1264,7 +1321,9 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
service_name = kwargs.get("service_name") or "my tags"
|
||||
raw_list = list(tags) if isinstance(tags, (list, tuple)) else [str(tags)]
|
||||
tag_list = [str(t).strip().lower() for t in raw_list if isinstance(t, str) and str(t).strip()]
|
||||
tag_list = [
|
||||
str(t).strip().lower() for t in raw_list if isinstance(t, str) and str(t).strip()
|
||||
]
|
||||
if not tag_list:
|
||||
return False
|
||||
client.delete_tag(file_hash, tag_list, service_name)
|
||||
@@ -1274,13 +1333,9 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
|
||||
def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]:
|
||||
"""Get known url for a Hydrus file.
|
||||
"""
|
||||
"""Get known url for a Hydrus file."""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
debug(f"{self._log_prefix()} get_url: client unavailable")
|
||||
return []
|
||||
|
||||
file_hash = str(file_identifier or "").strip().lower()
|
||||
if len(file_hash) != 64 or not all(ch in "0123456789abcdef" for ch in file_hash):
|
||||
@@ -1292,12 +1347,7 @@ class HydrusNetwork(Store):
|
||||
return []
|
||||
meta = items[0] if isinstance(items[0], dict) else {}
|
||||
|
||||
raw_urls: Any = (
|
||||
meta.get("known_urls")
|
||||
or meta.get("urls")
|
||||
or meta.get("url")
|
||||
or []
|
||||
)
|
||||
raw_urls: Any = meta.get("known_urls") or meta.get("urls") or meta.get("url") or []
|
||||
if isinstance(raw_urls, str):
|
||||
val = raw_urls.strip()
|
||||
return [val] if val else []
|
||||
@@ -1316,8 +1366,7 @@ class HydrusNetwork(Store):
|
||||
return []
|
||||
|
||||
def add_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool:
|
||||
"""Associate one or more url with a Hydrus file.
|
||||
"""
|
||||
"""Associate one or more url with a Hydrus file."""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
@@ -1344,11 +1393,11 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
|
||||
any_success = False
|
||||
for file_identifier, urls in (items or []):
|
||||
for file_identifier, urls in items or []:
|
||||
h = str(file_identifier or "").strip().lower()
|
||||
if len(h) != 64:
|
||||
continue
|
||||
for u in (urls or []):
|
||||
for u in urls or []:
|
||||
s = str(u or "").strip()
|
||||
if not s:
|
||||
continue
|
||||
@@ -1363,8 +1412,7 @@ class HydrusNetwork(Store):
|
||||
return False
|
||||
|
||||
def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool:
|
||||
"""Delete one or more url from a Hydrus file.
|
||||
"""
|
||||
"""Delete one or more url from a Hydrus file."""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
@@ -1453,35 +1501,35 @@ class HydrusNetwork(Store):
|
||||
|
||||
@staticmethod
|
||||
def _extract_tags_from_hydrus_meta(
|
||||
meta: Dict[str, Any],
|
||||
service_key: Optional[str],
|
||||
service_name: str
|
||||
meta: Dict[str, Any], service_key: Optional[str], service_name: str
|
||||
) -> List[str]:
|
||||
"""Extract current tags from Hydrus metadata dict.
|
||||
|
||||
|
||||
Prefers display_tags (includes siblings/parents, excludes deleted).
|
||||
Falls back to storage_tags status '0' (current).
|
||||
"""
|
||||
tags_payload = meta.get("tags")
|
||||
if not isinstance(tags_payload, dict):
|
||||
return []
|
||||
|
||||
|
||||
svc_data = None
|
||||
if service_key:
|
||||
svc_data = tags_payload.get(service_key)
|
||||
if not isinstance(svc_data, dict):
|
||||
return []
|
||||
|
||||
|
||||
# Prefer display_tags (Hydrus computes siblings/parents)
|
||||
display = svc_data.get("display_tags")
|
||||
if isinstance(display, list) and display:
|
||||
return [str(t) for t in display if isinstance(t, (str, bytes)) and str(t).strip()]
|
||||
|
||||
|
||||
# Fallback to storage_tags status '0' (current)
|
||||
storage = svc_data.get("storage_tags")
|
||||
if isinstance(storage, dict):
|
||||
current_list = storage.get("0") or storage.get(0)
|
||||
if isinstance(current_list, list):
|
||||
return [str(t) for t in current_list if isinstance(t, (str, bytes)) and str(t).strip()]
|
||||
|
||||
return [
|
||||
str(t) for t in current_list if isinstance(t, (str, bytes)) and str(t).strip()
|
||||
]
|
||||
|
||||
return []
|
||||
|
||||
@@ -57,7 +57,7 @@ class Store(ABC):
|
||||
Default behavior is to call add_url() per file.
|
||||
"""
|
||||
changed_any = False
|
||||
for file_identifier, urls in (items or []):
|
||||
for file_identifier, urls in items or []:
|
||||
try:
|
||||
ok = self.add_url(file_identifier, urls, **kwargs)
|
||||
changed_any = changed_any or bool(ok)
|
||||
@@ -72,7 +72,7 @@ class Store(ABC):
|
||||
Default behavior is to call delete_url() per file.
|
||||
"""
|
||||
changed_any = False
|
||||
for file_identifier, urls in (items or []):
|
||||
for file_identifier, urls in items or []:
|
||||
try:
|
||||
ok = self.delete_url(file_identifier, urls, **kwargs)
|
||||
changed_any = changed_any or bool(ok)
|
||||
@@ -87,7 +87,7 @@ class Store(ABC):
|
||||
Default behavior is to call set_note() per file.
|
||||
"""
|
||||
changed_any = False
|
||||
for file_identifier, name, text in (items or []):
|
||||
for file_identifier, name, text in items or []:
|
||||
try:
|
||||
ok = self.set_note(file_identifier, name, text, **kwargs)
|
||||
changed_any = changed_any or bool(ok)
|
||||
@@ -112,7 +112,9 @@ class Store(ABC):
|
||||
"""Add or replace a named note for a file."""
|
||||
raise NotImplementedError
|
||||
|
||||
def selector(self, selected_items: List[Any], *, ctx: Any, stage_is_last: bool = True, **_kwargs: Any) -> bool:
|
||||
def selector(
|
||||
self, selected_items: List[Any], *, ctx: Any, stage_is_last: bool = True, **_kwargs: Any
|
||||
) -> bool:
|
||||
"""Optional hook for handling `@N` selection semantics.
|
||||
|
||||
Return True if the selection was handled and default behavior should be skipped.
|
||||
|
||||
@@ -88,7 +88,9 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
|
||||
raise TypeError(f"Unsupported __new__.keys type for {store_cls.__name__}: {type(keys)}")
|
||||
|
||||
|
||||
def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]:
|
||||
def _build_kwargs(
|
||||
store_cls: Type[BaseStore], instance_name: str, instance_config: Any
|
||||
) -> Dict[str, Any]:
|
||||
if isinstance(instance_config, dict):
|
||||
cfg_dict = dict(instance_config)
|
||||
else:
|
||||
@@ -97,7 +99,10 @@ def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_confi
|
||||
required = _required_keys_for(store_cls)
|
||||
|
||||
# If NAME is required but not present, allow the instance key to provide it.
|
||||
if any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None:
|
||||
if (
|
||||
any(_normalize_config_key(k) == "NAME" for k in required)
|
||||
and _get_case_insensitive(cfg_dict, "NAME") is None
|
||||
):
|
||||
cfg_dict["NAME"] = str(instance_name)
|
||||
|
||||
kwargs: Dict[str, Any] = {}
|
||||
@@ -116,14 +121,18 @@ def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_confi
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None, suppress_debug: bool = False) -> None:
|
||||
def __init__(
|
||||
self, config: Optional[Dict[str, Any]] = None, suppress_debug: bool = False
|
||||
) -> None:
|
||||
self._config = config or {}
|
||||
self._suppress_debug = suppress_debug
|
||||
self._backends: Dict[str, BaseStore] = {}
|
||||
self._backend_errors: Dict[str, str] = {}
|
||||
self._load_backends()
|
||||
|
||||
def _maybe_register_temp_alias(self, store_type: str, backend_name: str, kwargs: Dict[str, Any], backend: BaseStore) -> None:
|
||||
def _maybe_register_temp_alias(
|
||||
self, store_type: str, backend_name: str, kwargs: Dict[str, Any], backend: BaseStore
|
||||
) -> None:
|
||||
"""If a folder backend points at config['temp'], also expose it as the 'temp' backend.
|
||||
|
||||
This keeps config compatibility (e.g. existing 'default') while presenting the temp
|
||||
@@ -236,7 +245,9 @@ class Store:
|
||||
|
||||
def __getitem__(self, backend_name: str) -> BaseStore:
|
||||
if backend_name not in self._backends:
|
||||
raise KeyError(f"Unknown store backend: {backend_name}. Available: {list(self._backends.keys())}")
|
||||
raise KeyError(
|
||||
f"Unknown store backend: {backend_name}. Available: {list(self._backends.keys())}"
|
||||
)
|
||||
return self._backends[backend_name]
|
||||
|
||||
def is_available(self, backend_name: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user