goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updated old legacy store names

Browse Source
This commit is contained in:
Nose
2026-05-23 13:49:47 -07:00
parent e8913d1344
commit a8cdd09e1e
31 changed files with 466 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Store/_base.py → PluginCore/backend_base.py
+14 -14
View File
@@ -1,6 +1,7 @@
"""Store backend base types. """Plugin-backed configured backend base types.
Concrete store implementations live in the `Store/` package. Concrete storage backends live with their owning plugins and are resolved
through PluginCore.backend_registry.
""" """
from __future__ import annotations from __future__ import annotations
@@ -10,11 +11,11 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
class Store(ABC): class BackendBase(ABC):
@classmethod @classmethod
def config_schema(cls) -> List[Dict[str, Any]]: def config_schema(cls) -> List[Dict[str, Any]]:
"""Return configuration schema for this store. """Return configuration schema for this backend.
Returns a list of dicts: Returns a list of dicts:
{ {
@@ -32,27 +33,27 @@ class Store(ABC):
@property @property
def is_remote(self) -> bool: def is_remote(self) -> bool:
"""True if the store is a remote service (e.g. Hydrus) rather than local disk.""" """True if this backend is a remote service rather than local disk."""
return False return False
@property @property
def prefer_defer_tags(self) -> bool: def prefer_defer_tags(self) -> bool:
"""True if the store prefers tags to be applied after the file is added.""" """True if this backend prefers tags to be applied after add_file()."""
return False return False
@property @property
def supports_url_association(self) -> bool: def supports_url_association(self) -> bool:
"""True when this store supports associating URLs to files.""" """True when this backend supports associating URLs to files."""
return False return False
@property @property
def supports_note_association(self) -> bool: def supports_note_association(self) -> bool:
"""True when this store supports per-file named notes.""" """True when this backend supports per-file named notes."""
return False return False
@property @property
def supports_relationship_association(self) -> bool: def supports_relationship_association(self) -> bool:
"""True when this store supports file relationship links (king/alt/related).""" """True when this backend supports file relationship links."""
return False return False
@abstractmethod @abstractmethod
@@ -111,9 +112,8 @@ class Store(ABC):
def delete_url_bulk( def delete_url_bulk(
self, self,
items: List[Tuple[str, items: List[Tuple[str, List[str]]],
List[str]]], **kwargs: Any,
**kwargs: Any
) -> bool: ) -> bool:
"""Optional bulk url deletion. """Optional bulk url deletion.
@@ -162,7 +162,7 @@ class Store(ABC):
file_identifier: str, file_identifier: str,
name: str, name: str,
text: str, text: str,
**kwargs: Any **kwargs: Any,
) -> bool: ) -> bool:
"""Add or replace a named note for a file.""" """Add or replace a named note for a file."""
raise NotImplementedError raise NotImplementedError
@@ -173,7 +173,7 @@ class Store(ABC):
*, *,
ctx: Any, ctx: Any,
stage_is_last: bool = True, stage_is_last: bool = True,
**_kwargs: Any **_kwargs: Any,
) -> bool: ) -> bool:
"""Optional hook for handling `@N` selection semantics. """Optional hook for handling `@N` selection semantics.
Store/registry.py → PluginCore/backend_registry.py
+113 -213
View File
@@ -1,38 +1,31 @@
"""Store registry. """Configured plugin-backed backend registry.
Concrete store implementations live in the `Store/` package. Backends are discovered from their owning plugins and instantiated from config.
This module is the single source of truth for store discovery.
This registry is config-driven:
- Each store subtype (e.g. `hydrusnetwork`) maps to a concrete store class.
- Each store class advertises its required config keys via `StoreClass.__new__.keys`.
- Instances are created from config using those keys (case-insensitive lookup).
""" """
from __future__ import annotations from __future__ import annotations
import importlib import importlib
import inspect import inspect
import pkgutil
import re import re
from typing import Any, Dict, Optional, Type from typing import Any, Dict, Optional, Type
from SYS.logger import debug from SYS.logger import debug
from SYS.utils import expand_path from SYS.utils import expand_path
from Store._base import Store as BaseStore from PluginCore.backend_base import BackendBase
_SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$") _SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$")
_DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None _PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BackendBase]]] = {}
_PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BaseStore]]] = {}
# Backends that failed to initialize earlier in the current process. # Backends that failed to initialize earlier in the current process.
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>. # Keyed by (backend_type, instance_key) where instance_key is the configured name
# under config.store.<type>.<instance_key>.
_FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {} _FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {}
def _normalize_store_type(value: str) -> str: def _normalize_backend_type(value: str) -> str:
return "".join(ch.lower() for ch in str(value or "") if ch.isalnum()) return "".join(ch.lower() for ch in str(value or "") if ch.isalnum())
@@ -44,59 +37,23 @@ def _get_case_insensitive(mapping: Dict[str, Any], key: str) -> Any:
if key in mapping: if key in mapping:
return mapping[key] return mapping[key]
desired = _normalize_config_key(key) desired = _normalize_config_key(key)
for k, v in mapping.items(): for current_key, value in mapping.items():
if _normalize_config_key(k) == desired: if _normalize_config_key(current_key) == desired:
return v return value
return None return None
def _discover_store_classes() -> Dict[str, Type[BaseStore]]: def _extract_backend_classes(owner: Any) -> Dict[str, Type[BackendBase]]:
"""Discover store classes from the Store package. discovered: Dict[str, Type[BackendBase]] = {}
Convention:
- The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork).
"""
global _DISCOVERED_CLASSES_CACHE
if _DISCOVERED_CLASSES_CACHE is not None:
return _DISCOVERED_CLASSES_CACHE
import Store as store_pkg
discovered: Dict[str, Type[BaseStore]] = {}
for module_info in pkgutil.iter_modules(store_pkg.__path__):
module_name = module_info.name
if module_name.startswith(("_", "registry")):
continue
try:
module = importlib.import_module(f"Store.{module_name}")
for _, obj in vars(module).items():
if not inspect.isclass(obj):
continue
if obj is BaseStore:
continue
if not issubclass(obj, BaseStore):
continue
discovered[_normalize_store_type(obj.__name__)] = obj
except Exception as exc:
debug(f"[Store] Failed to import module '{module_name}': {exc}")
continue
_DISCOVERED_CLASSES_CACHE = discovered
return discovered
def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]:
discovered: Dict[str, Type[BaseStore]] = {}
def _add_candidate(key: Any, candidate: Any) -> None: def _add_candidate(key: Any, candidate: Any) -> None:
if not inspect.isclass(candidate): if not inspect.isclass(candidate):
return return
if candidate is BaseStore: if candidate is BackendBase:
return return
if not issubclass(candidate, BaseStore): if not issubclass(candidate, BackendBase):
return return
normalized = _normalize_store_type(str(key or candidate.__name__)) normalized = _normalize_backend_type(str(key or candidate.__name__))
if normalized: if normalized:
discovered[normalized] = candidate discovered[normalized] = candidate
@@ -125,8 +82,8 @@ def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]:
return discovered return discovered
def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]: def _discover_plugin_backend_class(backend_type: str) -> Optional[Type[BackendBase]]:
normalized = _normalize_store_type(store_type) normalized = _normalize_backend_type(backend_type)
if not normalized: if not normalized:
return None return None
@@ -140,19 +97,19 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]:
_PLUGIN_DISCOVERED_CLASSES_CACHE[normalized] = None _PLUGIN_DISCOVERED_CLASSES_CACHE[normalized] = None
return None return None
discovered: Dict[str, Type[BaseStore]] = {} discovered: Dict[str, Type[BackendBase]] = {}
backend_hook = getattr(plugin_module, "get_store_backend_classes", None) backend_hook = getattr(plugin_module, "get_store_backend_classes", None)
if callable(backend_hook): if callable(backend_hook):
try: try:
discovered.update(_extract_store_classes(backend_hook())) discovered.update(_extract_backend_classes(backend_hook()))
except Exception as exc: except Exception as exc:
debug(f"[Store] Failed to load plugin store backends for '{normalized}': {exc}") debug(f"[BackendRegistry] Failed to load plugin backends for '{normalized}': {exc}")
discovered.update(_extract_store_classes(getattr(plugin_module, "STORE_BACKENDS", None))) discovered.update(_extract_backend_classes(getattr(plugin_module, "STORE_BACKENDS", None)))
if normalized not in discovered: if normalized not in discovered:
discovered.update(_extract_store_classes(plugin_module)) discovered.update(_extract_backend_classes(plugin_module))
resolved = discovered.get(normalized) resolved = discovered.get(normalized)
if resolved is None and len(discovered) == 1: if resolved is None and len(discovered) == 1:
@@ -162,37 +119,24 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]:
return resolved return resolved
def _resolve_store_class( def _resolve_backend_class(backend_type: str) -> Optional[Type[BackendBase]]:
store_type: str, normalized = _normalize_backend_type(backend_type)
classes_by_type: Optional[Dict[str, Type[BaseStore]]] = None,
) -> Optional[Type[BaseStore]]:
normalized = _normalize_store_type(store_type)
if not normalized: if not normalized:
return None return None
return _discover_plugin_backend_class(normalized)
plugin_resolved = _discover_plugin_store_class(normalized)
if plugin_resolved is not None:
return plugin_resolved
discovered = classes_by_type if classes_by_type is not None else _discover_store_classes()
resolved = discovered.get(normalized)
if resolved is not None:
return resolved
return None
def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]: def _required_keys_for(backend_cls: Type[BackendBase]) -> list[str]:
# Support new config_schema() schema if hasattr(backend_cls, "config_schema") and callable(backend_cls.config_schema):
if hasattr(store_cls, "config_schema") and callable(store_cls.config_schema):
try: try:
schema = store_cls.config_schema() schema = backend_cls.config_schema()
keys = [] keys = []
if isinstance(schema, list): if isinstance(schema, list):
for field in schema: for field in schema:
if isinstance(field, dict) and field.get("required"): if isinstance(field, dict) and field.get("required"):
k = field.get("key") key = field.get("key")
if k: if key:
keys.append(str(k)) keys.append(str(key))
if keys: if keys:
return keys return keys
except Exception: except Exception:
@@ -200,21 +144,14 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
return [] return []
# Store type names that have been converted to providers-only. _PROVIDER_ONLY_BACKEND_NAMES = frozenset(("debrid", "alldebrid"))
# These should be silently skipped without warning.
_PROVIDER_ONLY_STORE_NAMES = frozenset(("debrid", "alldebrid"))
def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]: def _build_kwargs(backend_cls: Type[BackendBase], instance_name: str, instance_config: Any) -> Dict[str, Any]:
if isinstance(instance_config, dict): cfg_dict = dict(instance_config) if isinstance(instance_config, dict) else {}
cfg_dict = dict(instance_config)
else:
cfg_dict = {}
required = _required_keys_for(store_cls) required = _required_keys_for(backend_cls)
if any(_normalize_config_key(key) == "NAME" for key in required) and _get_case_insensitive(cfg_dict, "NAME") is None:
# If NAME is required but not present, allow the instance key to provide it.
if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None):
cfg_dict["NAME"] = str(instance_name) cfg_dict["NAME"] = str(instance_name)
kwargs: Dict[str, Any] = {} kwargs: Dict[str, Any] = {}
@@ -228,28 +165,24 @@ def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_confi
if missing: if missing:
raise ValueError( raise ValueError(
f"Missing required keys for {store_cls.__name__}: {', '.join(missing)}" f"Missing required keys for {backend_cls.__name__}: {', '.join(missing)}"
) )
return kwargs return kwargs
class Store: class BackendRegistry:
def __init__( def __init__(
self, self,
config: Optional[Dict[str, config: Optional[Dict[str, Any]] = None,
Any]] = None, suppress_debug: bool = False,
suppress_debug: bool = False
) -> None: ) -> None:
self._config = config or {} self._config = config or {}
self._suppress_debug = suppress_debug self._suppress_debug = suppress_debug
self._backends: Dict[str, self._backends: Dict[str, BackendBase] = {}
BaseStore] = {} self._backend_errors: Dict[str, str] = {}
self._backend_errors: Dict[str, self._backend_types: Dict[str, str] = {}
str] = {}
self._backend_types: Dict[str,
str] = {}
self._load_backends() self._load_backends()
def _load_backends(self) -> None: def _load_backends(self) -> None:
@@ -258,61 +191,54 @@ class Store:
store_cfg = {} store_cfg = {}
self._backend_types = {} self._backend_types = {}
classes_by_type = _discover_store_classes() for raw_backend_type, instances in store_cfg.items():
for raw_store_type, instances in store_cfg.items():
if not isinstance(instances, dict): if not isinstance(instances, dict):
continue continue
store_type = _normalize_store_type(str(raw_store_type)) backend_type = _normalize_backend_type(str(raw_backend_type))
if store_type == "folder": if backend_type == "folder":
continue continue
store_cls = _resolve_store_class(store_type, classes_by_type) backend_cls = _resolve_backend_class(backend_type)
if store_cls is None: if backend_cls is None:
# Skip provider-only names without debug warning if backend_type not in _PROVIDER_ONLY_BACKEND_NAMES and not self._suppress_debug:
if store_type not in _PROVIDER_ONLY_STORE_NAMES and not self._suppress_debug: debug(f"[BackendRegistry] Unknown backend type '{raw_backend_type}'")
debug(f"[Store] Unknown store type '{raw_store_type}'")
continue continue
for instance_name, instance_config in instances.items(): for instance_name, instance_config in instances.items():
backend_name = str(instance_name) backend_name = str(instance_name)
cache_key = (backend_type, str(instance_name))
# If this backend already failed earlier in this process, skip re-instantiation.
cache_key = (store_type, str(instance_name))
cached_error = _FAILED_BACKEND_CACHE.get(cache_key) cached_error = _FAILED_BACKEND_CACHE.get(cache_key)
if cached_error: if cached_error:
self._backend_errors[str(instance_name)] = str(cached_error) self._backend_errors[str(instance_name)] = str(cached_error)
if isinstance(instance_config, dict): if isinstance(instance_config, dict):
override_name = _get_case_insensitive( override_name = _get_case_insensitive(dict(instance_config), "NAME")
dict(instance_config),
"NAME"
)
if override_name: if override_name:
self._backend_errors[str(override_name)] = str(cached_error) self._backend_errors[str(override_name)] = str(cached_error)
continue continue
try: try:
kwargs = _build_kwargs( kwargs = _build_kwargs(
store_cls, backend_cls,
str(instance_name), str(instance_name),
instance_config instance_config,
) )
# Convenience normalization for filesystem-like paths.
for key in list(kwargs.keys()): for key in list(kwargs.keys()):
if _normalize_config_key(key) in {"PATH", "LOCATION"}: if _normalize_config_key(key) in {"PATH", "LOCATION"}:
kwargs[key] = str(expand_path(kwargs[key])) kwargs[key] = str(expand_path(kwargs[key]))
backend = store_cls(**kwargs) backend = backend_cls(**kwargs)
backend_name = str(kwargs.get("NAME") or instance_name) backend_name = str(kwargs.get("NAME") or instance_name)
self._backends[backend_name] = backend self._backends[backend_name] = backend
self._backend_types[backend_name] = store_type self._backend_types[backend_name] = backend_type
except Exception as exc: except Exception as exc:
err_text = str(exc) err_text = str(exc)
self._backend_errors[str(instance_name)] = err_text self._backend_errors[str(instance_name)] = err_text
_FAILED_BACKEND_CACHE[cache_key] = err_text _FAILED_BACKEND_CACHE[cache_key] = err_text
if not self._suppress_debug: if not self._suppress_debug:
debug( debug(
f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}" f"[BackendRegistry] Failed to register {backend_cls.__name__} instance '{instance_name}': {exc}"
) )
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]: def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
@@ -320,46 +246,41 @@ class Store:
if requested in self._backends: if requested in self._backends:
return requested, None return requested, None
requested_norm = _normalize_store_type(requested) requested_norm = _normalize_backend_type(requested)
ci_matches = [ ci_matches = [
name for name in self._backends name for name in self._backends
if _normalize_store_type(name) == requested_norm if _normalize_backend_type(name) == requested_norm
] ]
if len(ci_matches) == 1: if len(ci_matches) == 1:
return ci_matches[0], None return ci_matches[0], None
if len(ci_matches) > 1: if len(ci_matches) > 1:
return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}" return None, f"Ambiguous backend alias '{backend_name}' matches {ci_matches}"
type_matches = [ type_matches = [
name for name, store_type in self._backend_types.items() name for name, backend_type in self._backend_types.items()
if store_type == requested_norm if backend_type == requested_norm
] ]
if len(type_matches) == 1: if len(type_matches) == 1:
return type_matches[0], None return type_matches[0], None
if len(type_matches) > 1: if len(type_matches) > 1:
return None, ( return None, (
f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}" f"Ambiguous backend alias '{backend_name}' matches type '{requested_norm}': {type_matches}"
) )
prefix_matches = [ prefix_matches = [
name for name, store_type in self._backend_types.items() name for name, backend_type in self._backend_types.items()
if store_type.startswith(requested_norm) if backend_type.startswith(requested_norm)
] ]
if len(prefix_matches) == 1: if len(prefix_matches) == 1:
return prefix_matches[0], None return prefix_matches[0], None
if len(prefix_matches) > 1: if len(prefix_matches) > 1:
return None, ( return None, (
f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}" f"Ambiguous backend alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}"
) )
return None, None return None, None
# get_backend_instance implementation moved to the bottom of this file to avoid
# instantiating all backends during startup (see function `get_backend_instance`).
# Duplicate _resolve_backend_name removed — the method is defined once earlier in the class.
def get_backend_error(self, backend_name: str) -> Optional[str]: def get_backend_error(self, backend_name: str) -> Optional[str]:
return self._backend_errors.get(str(backend_name)) return self._backend_errors.get(str(backend_name))
@@ -367,19 +288,17 @@ class Store:
return sorted(self._backends.keys()) return sorted(self._backends.keys())
def list_searchable_backends(self) -> list[str]: def list_searchable_backends(self) -> list[str]:
# De-duplicate backends by instance (aliases can point at the same object).
def _rank(name: str) -> int: def _rank(name: str) -> int:
n = str(name or "").strip().lower() normalized_name = str(name or "").strip().lower()
if n == "temp": if normalized_name == "temp":
return 0 return 0
if n == "default": if normalized_name == "default":
return 2 return 2
return 1 return 1
chosen: Dict[int, chosen: Dict[int, str] = {}
str] = {}
for name, backend in self._backends.items(): for name, backend in self._backends.items():
if type(backend).search is BaseStore.search: if type(backend).search is BackendBase.search:
continue continue
key = id(backend) key = id(backend)
prev = chosen.get(key) prev = chosen.get(key)
@@ -387,27 +306,20 @@ class Store:
chosen[key] = name chosen[key] = name
return sorted(chosen.values()) return sorted(chosen.values())
def __getitem__(self, backend_name: str) -> BaseStore: def __getitem__(self, backend_name: str) -> BackendBase:
resolved, err = self._resolve_backend_name(backend_name) resolved, err = self._resolve_backend_name(backend_name)
if resolved: if resolved:
return self._backends[resolved] return self._backends[resolved]
if err: if err:
raise KeyError( raise KeyError(f"Unknown backend: {backend_name}. {err}")
f"Unknown store backend: {backend_name}. {err}" raise KeyError(f"Unknown backend: {backend_name}. Available: {list(self._backends.keys())}")
)
raise KeyError(
f"Unknown store backend: {backend_name}. Available: {list(self._backends.keys())}"
)
def is_available(self, backend_name: str) -> bool: def is_available(self, backend_name: str) -> bool:
resolved, _err = self._resolve_backend_name(backend_name) resolved, _err = self._resolve_backend_name(backend_name)
return resolved is not None return resolved is not None
def try_add_url_for_pipe_object(self, pipe_obj: Any, url: str) -> bool: def try_add_url_for_pipe_object(self, pipe_obj: Any, url: str) -> bool:
"""Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that store backend. """Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that backend."""
Intended for providers to attach newly generated/hosted URLs back to an existing stored file.
"""
try: try:
url_text = str(url or "").strip() url_text = str(url or "").strip()
if not url_text: if not url_text:
@@ -442,41 +354,28 @@ class Store:
def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]: def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]:
"""Return backend instance names present in the provided config WITHOUT instantiating backends. """Return configured backend instance names without instantiating backends."""
This is a lightweight helper for CLI usage where we only need to know if a
configured backend exists (e.g., to distinguish a store name from a filesystem path)
without triggering backend initialization (which may perform network calls).
Behaviour:
- Only includes store types that map to a discovered save backend class.
- Skips folder/provider-only/unknown entries so UI pickers do not surface
non-ingest destinations such as provider helpers.
- For each configured store type, returns the per-instance NAME override
(case-insensitive) when present, otherwise the instance key.
"""
try: try:
classes_by_type = _discover_store_classes()
names: list[str] = [] names: list[str] = []
for section_name in ("store", "plugin", "provider"): for section_name in ("store", "plugin", "provider"):
section_cfg = (config or {}).get(section_name) or {} section_cfg = (config or {}).get(section_name) or {}
if not isinstance(section_cfg, dict): if not isinstance(section_cfg, dict):
continue continue
for raw_store_type, instances in section_cfg.items(): for raw_backend_type, instances in section_cfg.items():
if not isinstance(instances, dict): if not isinstance(instances, dict):
continue continue
store_type = _normalize_store_type(str(raw_store_type)) backend_type = _normalize_backend_type(str(raw_backend_type))
if store_type == "folder" or store_type in _PROVIDER_ONLY_STORE_NAMES: if backend_type == "folder" or backend_type in _PROVIDER_ONLY_BACKEND_NAMES:
continue continue
store_cls = _resolve_store_class(store_type, classes_by_type) backend_cls = _resolve_backend_class(backend_type)
if store_cls is None: if backend_cls is None:
continue continue
for instance_name, instance_config in instances.items(): for instance_name, instance_config in instances.items():
try: try:
_build_kwargs(store_cls, str(instance_name), instance_config) _build_kwargs(backend_cls, str(instance_name), instance_config)
except Exception: except Exception:
continue continue
if isinstance(instance_config, dict): if isinstance(instance_config, dict):
@@ -493,18 +392,15 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]
return [] return []
def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]: def get_backend_instance(
"""Instantiate and return a single store backend by configured name. config: Optional[Dict[str, Any]],
backend_name: str,
This avoids creating all configured backends (and opening their DBs) *,
when only a single backend is needed (common in `get-file`/`get-metadata`). suppress_debug: bool = False,
The function first tries a lightweight match against raw config NAME/value to ) -> Optional[BackendBase]:
avoid calling `_build_kwargs` (which can raise if keys are missing). """Instantiate and return a single configured backend by name."""
Returns None when no matching backend is found or instantiation fails.
"""
if not backend_name: if not backend_name:
return None return None
classes_by_type = _discover_store_classes()
desired = str(backend_name or "").strip().lower() desired = str(backend_name or "").strip().lower()
for section_name in ("store", "plugin", "provider"): for section_name in ("store", "plugin", "provider"):
@@ -512,29 +408,28 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
if not isinstance(section_cfg, dict): if not isinstance(section_cfg, dict):
continue continue
for raw_store_type, instances in section_cfg.items(): for raw_backend_type, instances in section_cfg.items():
if not isinstance(instances, dict): if not isinstance(instances, dict):
continue continue
store_type = _normalize_store_type(str(raw_store_type)) backend_type = _normalize_backend_type(str(raw_backend_type))
store_cls = _resolve_store_class(store_type, classes_by_type) backend_cls = _resolve_backend_class(backend_type)
if store_cls is None: if backend_cls is None:
continue continue
# Fast path: match using raw 'NAME' or 'name' in config without building full kwargs
for instance_name, instance_cfg in instances.items(): for instance_name, instance_cfg in instances.items():
candidate_alias = None candidate_alias = None
if isinstance(instance_cfg, dict): if isinstance(instance_cfg, dict):
candidate_alias = ( candidate_alias = instance_cfg.get("NAME") or instance_cfg.get("name")
instance_cfg.get("NAME") or instance_cfg.get("name")
)
candidate_alias = str(candidate_alias or instance_name).strip() candidate_alias = str(candidate_alias or instance_name).strip()
if candidate_alias.lower() != desired: if candidate_alias.lower() != desired:
continue continue
try: try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg)
except Exception as exc: except Exception as exc:
if not suppress_debug: if not suppress_debug:
debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}/{section_name}): {exc}") debug(
f"[BackendRegistry] Can't build kwargs for '{instance_name}' ({backend_type}/{section_name}): {exc}"
)
return None return None
try: try:
for key in list(kwargs.keys()): for key in list(kwargs.keys()):
@@ -543,17 +438,15 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
except Exception: except Exception:
pass pass
try: try:
backend = store_cls(**kwargs) return backend_cls(**kwargs)
return backend
except Exception as exc: except Exception as exc:
if not suppress_debug: if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}") debug(f"[BackendRegistry] Failed to instantiate backend '{candidate_alias}': {exc}")
return None return None
# Fallback: build kwargs for each instance and compare resolved NAME
for instance_name, instance_cfg in instances.items(): for instance_name, instance_cfg in instances.items():
try: try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg)
except Exception: except Exception:
continue continue
alias = str(kwargs.get("NAME") or instance_name).strip() alias = str(kwargs.get("NAME") or instance_name).strip()
@@ -566,13 +459,20 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
except Exception: except Exception:
pass pass
try: try:
backend = store_cls(**kwargs) return backend_cls(**kwargs)
return backend
except Exception as exc: except Exception as exc:
if not suppress_debug: if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{alias}': {exc}") debug(f"[BackendRegistry] Failed to instantiate backend '{alias}': {exc}")
return None return None
if not suppress_debug: if not suppress_debug:
debug(f"[Store] Backend '{backend_name}' not found in config") debug(f"[BackendRegistry] Backend '{backend_name}' not found in config")
return None return None
__all__ = [
"BackendBase",
"BackendRegistry",
"get_backend_instance",
"list_configured_backend_names",
]
PluginCore/base.py
+1 -1
View File
@@ -621,7 +621,7 @@ class Provider(ABC):
raise NotImplementedError(f"Plugin '{self.name}' does not support upload") raise NotImplementedError(f"Plugin '{self.name}' does not support upload")
# ----------------------------------------------------------------------- # -----------------------------------------------------------------------
# Storage interface — mirrors Store._base.Store. # Storage interface — mirrors PluginCore.backend_base.BackendBase.
# Plugins that act as file repositories override these methods. # Plugins that act as file repositories override these methods.
# All raise NotImplementedError by default; override selectively. # All raise NotImplementedError by default; override selectively.
# ----------------------------------------------------------------------- # -----------------------------------------------------------------------
SYS/cmdlet_spec.py
+2 -2
View File
@@ -125,9 +125,9 @@ class SharedArgs:
return return
try: try:
from Store.registry import Store as StoreRegistry from PluginCore.backend_registry import BackendRegistry
registry = StoreRegistry(config=config, suppress_debug=True) registry = BackendRegistry(config=config, suppress_debug=True)
available = registry.list_backends() available = registry.list_backends()
if available: if available:
SharedArgs._cached_available_stores = available SharedArgs._cached_available_stores = available
SYS/pipeline.py
+6 -6
View File
@@ -1730,21 +1730,21 @@ class PipelineExecutor:
if store_keys: if store_keys:
try: try:
from Store.registry import Store as StoreRegistry from PluginCore.backend_registry import BackendRegistry
store_registry = StoreRegistry(config, suppress_debug=True) backend_registry = BackendRegistry(config, suppress_debug=True)
_backend_names = list(store_registry.list_backends() or []) _backend_names = list(backend_registry.list_backends() or [])
_backend_by_lower = { _backend_by_lower = {
str(n).lower(): str(n) str(n).lower(): str(n)
for n in _backend_names if str(n).strip() for n in _backend_names if str(n).strip()
} }
for name in store_keys: for name in store_keys:
resolved_name = name resolved_name = name
if not store_registry.is_available(resolved_name): if not backend_registry.is_available(resolved_name):
resolved_name = _backend_by_lower.get(str(name).lower(), name) resolved_name = _backend_by_lower.get(str(name).lower(), name)
if not store_registry.is_available(resolved_name): if not backend_registry.is_available(resolved_name):
continue continue
backend = store_registry[resolved_name] backend = backend_registry[resolved_name]
selector = getattr(backend, "selector", None) selector = getattr(backend, "selector", None)
if selector is None: if selector is None:
continue continue
Store/__init__.py
-7
View File
@@ -1,7 +0,0 @@
from Store._base import Store as BaseStore
from Store.registry import Store
__all__ = [
"Store",
"BaseStore",
]
TUI.py
+5 -5
View File
@@ -53,7 +53,7 @@ from SYS.result_table import Table, extract_hash_value, extract_store_value, get
from SYS.config import load_config # type: ignore # noqa: E402 from SYS.config import load_config # type: ignore # noqa: E402
from SYS.database import db from SYS.database import db
from Store.registry import Store as StoreRegistry # type: ignore # noqa: E402 from PluginCore.backend_registry import BackendRegistry # type: ignore # noqa: E402
from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402 from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402
from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402 from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402
@@ -261,7 +261,7 @@ class TagEditorPopup(ModalScreen[None]):
if not store_key or not hash_value: if not store_key or not hash_value:
return None return None
try: try:
registry = StoreRegistry(config=cfg, suppress_debug=True) registry = BackendRegistry(config=cfg, suppress_debug=True)
except Exception: except Exception:
return [] return []
match = None match = None
@@ -984,9 +984,9 @@ class PipelineHubApp(App):
stores: List[str] = [] stores: List[str] = []
try: try:
stores = StoreRegistry(config=cfg, suppress_debug=True).list_backends() stores = BackendRegistry(config=cfg, suppress_debug=True).list_backends()
except Exception: except Exception:
logger.exception("Failed to list store backends from StoreRegistry") logger.exception("Failed to list store backends from BackendRegistry")
stores = [] stores = []
# Always offer a reasonable default even if config is missing. # Always offer a reasonable default even if config is missing.
@@ -1380,7 +1380,7 @@ class PipelineHubApp(App):
cfg = {} cfg = {}
try: try:
registry = StoreRegistry(config=cfg, suppress_debug=True) registry = BackendRegistry(config=cfg, suppress_debug=True)
except Exception: except Exception:
return None return None
cmdlet/_shared.py
+3 -3
View File
@@ -1084,9 +1084,9 @@ def get_store_backend(
registry = store_registry registry = store_registry
if registry is None: if registry is None:
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
registry = Store(config or {}, suppress_debug=suppress_debug) registry = BackendRegistry(config or {}, suppress_debug=suppress_debug)
except Exception as exc: except Exception as exc:
return None, None, exc return None, None, exc
@@ -1110,7 +1110,7 @@ def get_preferred_store_backend(
"""Prefer a targeted backend instance before falling back to registry lookup.""" """Prefer a targeted backend instance before falling back to registry lookup."""
direct_exc: Optional[Exception] = None direct_exc: Optional[Exception] = None
try: try:
from Store.registry import get_backend_instance from PluginCore.backend_registry import get_backend_instance
backend = get_backend_instance( backend = get_backend_instance(
config or {}, config or {},
cmdlet/delete_url.py
+2 -2
View File
@@ -14,7 +14,7 @@ from ._shared import (
normalize_hash, normalize_hash,
) )
from SYS.logger import log from SYS.logger import log
from Store import Store from PluginCore.backend_registry import BackendRegistry
class Delete_Url(Cmdlet): class Delete_Url(Cmdlet):
@@ -106,7 +106,7 @@ class Delete_Url(Cmdlet):
# Get backend and delete url # Get backend and delete url
try: try:
storage = Store(config) storage = BackendRegistry(config)
store_override = parsed.get("instance") store_override = parsed.get("instance")
cmdlet/file/add.py
+66 -93
View File
@@ -16,7 +16,7 @@ from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store from PluginCore.backend_registry import BackendRegistry
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
from .. import _shared as sh from .. import _shared as sh
@@ -45,21 +45,21 @@ SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS
class _CommandDependencies: class _CommandDependencies:
"""Command-scope cache for Store and plugin instances to avoid repeated instantiation.""" """Command-scope cache for the backend registry and plugin instances."""
def __init__(self, config: Dict[str, Any]) -> None: def __init__(self, config: Dict[str, Any]) -> None:
self.config = config self.config = config
self._store: Optional[Store] = None self._backend_registry: Optional[BackendRegistry] = None
self._plugins: Dict[str, Any] = {} self._plugins: Dict[str, Any] = {}
def get_store(self) -> Optional[Store]: def get_backend_registry(self) -> Optional[BackendRegistry]:
"""Lazily initialize and return the command-scope Store instance.""" """Lazily initialize and return the command-scope backend registry."""
if self._store is None: if self._backend_registry is None:
try: try:
self._store = Store(self.config) self._backend_registry = BackendRegistry(self.config)
except Exception: except Exception:
self._store = None self._backend_registry = None
return self._store return self._backend_registry
def get_plugin(self, name: str) -> Optional[Any]: def get_plugin(self, name: str) -> Optional[Any]:
"""Cached plugin lookup by name.""" """Cached plugin lookup by name."""
@@ -236,7 +236,7 @@ class Add_File(Cmdlet):
# Initialize command-scope dependency context (caches Store/plugins) # Initialize command-scope dependency context (caches Store/plugins)
deps = _CommandDependencies(config) deps = _CommandDependencies(config)
storage_registry = deps.get_store() storage_registry = deps.get_backend_registry()
path_arg = parsed.get("path") path_arg = parsed.get("path")
location = parsed.get("instance") location = parsed.get("instance")
@@ -354,8 +354,8 @@ class Add_File(Cmdlet):
is_storage_backend_location = False is_storage_backend_location = False
if location: if location:
try: try:
store_for_lookup = storage_registry or deps.get_store() backend_registry_for_lookup = storage_registry or deps.get_backend_registry()
is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None is_storage_backend_location = Add_File._resolve_backend_by_name(backend_registry_for_lookup, str(location)) is not None
except Exception: except Exception:
is_storage_backend_location = False is_storage_backend_location = False
@@ -697,8 +697,8 @@ class Add_File(Cmdlet):
if location: if location:
try: try:
store = storage_registry or deps.get_store() backend_registry = storage_registry or deps.get_backend_registry()
resolved_backend = Add_File._resolve_backend_by_name(store, str(location)) resolved_backend = Add_File._resolve_backend_by_name(backend_registry, str(location))
if resolved_backend is not None: if resolved_backend is not None:
code = self._handle_storage_backend( code = self._handle_storage_backend(
item, item,
@@ -845,7 +845,7 @@ class Add_File(Cmdlet):
hash_values: List[str], hash_values: List[str],
config: Dict[str, config: Dict[str,
Any], Any],
store_instance: Optional[Store] = None, store_instance: Optional[BackendRegistry] = None,
) -> Optional[List[Any]]: ) -> Optional[List[Any]]:
"""Run search-file for a list of hashes and promote the table to a display overlay. """Run search-file for a list of hashes and promote the table to a display overlay.
@@ -1053,7 +1053,7 @@ class Add_File(Cmdlet):
str]]], str]]],
config: Dict[str, config: Dict[str,
Any], Any],
store_instance: Optional[Store] = None, store_instance: Optional[BackendRegistry] = None,
deps: Optional[_CommandDependencies] = None, deps: Optional[_CommandDependencies] = None,
) -> None: ) -> None:
"""Persist relationships to backends that support relationships. """Persist relationships to backends that support relationships.
@@ -1067,7 +1067,7 @@ class Add_File(Cmdlet):
deps = _CommandDependencies(config) deps = _CommandDependencies(config)
try: try:
store = store_instance if store_instance is not None else deps.get_store() backend_registry = store_instance if store_instance is not None else deps.get_backend_registry()
except Exception: except Exception:
return return
@@ -1076,7 +1076,7 @@ class Add_File(Cmdlet):
continue continue
try: try:
backend = store[str(backend_name)] backend = backend_registry[str(backend_name)]
except Exception: except Exception:
continue continue
@@ -1356,9 +1356,9 @@ class Add_File(Cmdlet):
try: try:
if deps is None: if deps is None:
deps = _CommandDependencies(config) deps = _CommandDependencies(config)
store = store_instance or deps.get_store() backend_registry = store_instance or deps.get_backend_registry()
backend = Add_File._resolve_backend_by_name(store, r_store) backend = Add_File._resolve_backend_by_name(backend_registry, r_store)
if backend is not None: if backend is not None:
mp = backend.get_file(r_hash) mp = backend.get_file(r_hash)
if isinstance(mp, Path) and mp.exists(): if isinstance(mp, Path) and mp.exists():
@@ -1497,14 +1497,14 @@ class Add_File(Cmdlet):
explicit_instance = str(instance_name or "").strip() or None explicit_instance = str(instance_name or "").strip() or None
try: try:
storage = store_instance if store_instance is not None else Store(config) backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception: except Exception:
storage = None backend_registry = None
try: try:
resolved_name, backend = resolver( resolved_name, backend = resolver(
explicit_instance, explicit_instance,
storage=storage, storage=backend_registry,
require_explicit=bool(explicit_instance), require_explicit=bool(explicit_instance),
) )
except TypeError: except TypeError:
@@ -1622,8 +1622,8 @@ class Add_File(Cmdlet):
if deps is None: if deps is None:
deps = _CommandDependencies(config) deps = _CommandDependencies(config)
store = store_instance or deps.get_store() backend_registry = store_instance or deps.get_backend_registry()
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None backend = Add_File._resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None
if backend is None: if backend is None:
return None, None, None return None, None, None
@@ -2475,10 +2475,23 @@ class Add_File(Cmdlet):
List[str]]]]] = None, List[str]]]]] = None,
suppress_last_stage_overlay: bool = False, suppress_last_stage_overlay: bool = False,
auto_search_file: bool = True, auto_search_file: bool = True,
store_instance: Optional[Store] = None, store_instance: Optional[BackendRegistry] = None,
) -> int: ) -> int:
"""Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.).""" """Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.)."""
##log(f"Adding file to storage backend '{backend_name}': {media_path.name}", file=sys.stderr) ##log(f"Adding file to storage backend '{backend_name}': {media_path.name}", file=sys.stderr)
pipeline_progress = PipelineProgress(ctx)
def _set_status(text: str) -> None:
try:
pipeline_progress.set_status(f"{backend_name}: {text}")
except Exception:
pass
def _clear_status() -> None:
try:
pipeline_progress.clear_status()
except Exception:
pass
delete_after_effective = bool(delete_after) delete_after_effective = bool(delete_after)
# ... (lines omitted for brevity but I need to keep them contextually correct) # ... (lines omitted for brevity but I need to keep them contextually correct)
@@ -2510,11 +2523,11 @@ class Add_File(Cmdlet):
pass pass
try: try:
store = store_instance if store_instance is not None else Store(config) backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
backend, store, backend_exc = sh.get_preferred_store_backend( backend, backend_registry, backend_exc = sh.get_preferred_store_backend(
config, config,
backend_name, backend_name,
store_registry=store, store_registry=backend_registry,
suppress_debug=True, suppress_debug=True,
) )
if backend is None: if backend is None:
@@ -2654,6 +2667,8 @@ class Add_File(Cmdlet):
tag=upload_tags, tag=upload_tags,
url=[] if ((defer_url_association and url) or (not supports_url_association)) else url, url=[] if ((defer_url_association and url) or (not supports_url_association)) else url,
file_hash=f_hash, file_hash=f_hash,
pipeline_progress=pipeline_progress,
transfer_label=title or media_path.name,
) )
##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr) ##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr)
@@ -2704,6 +2719,7 @@ class Add_File(Cmdlet):
try: try:
adder = getattr(backend, "add_tag", None) adder = getattr(backend, "add_tag", None)
if callable(adder): if callable(adder):
_set_status("applying deferred tags")
adder(resolved_hash, list(tags)) adder(resolved_hash, list(tags))
except Exception as exc: except Exception as exc:
log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr) log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr)
@@ -2724,88 +2740,43 @@ class Add_File(Cmdlet):
try: try:
# Folder.add_file already persists URLs, avoid extra DB traffic here. # Folder.add_file already persists URLs, avoid extra DB traffic here.
if not is_folder_backend: if not is_folder_backend:
_set_status("associating urls")
backend.add_url(resolved_hash, list(url)) backend.add_url(resolved_hash, list(url))
except Exception: except Exception:
pass pass
# If a subtitle note was provided upstream (e.g., download-media writes notes.sub), # If a subtitle note was provided upstream (e.g., download-media writes notes.sub),
# persist it automatically like add-note would. # persist it automatically like add-note would.
sub_note = Add_File._get_note_text(result, pipe_obj, "sub") def _write_note(note_name: str, note_text: Optional[str]) -> None:
if sub_note and supports_note_association: if not note_text or not supports_note_association:
return
try: try:
setter = getattr(backend, "set_note", None) setter = getattr(backend, "set_note", None)
if callable(setter): if callable(setter):
setter(resolved_hash, "sub", sub_note) _set_status(f"writing {note_name} note")
setter(resolved_hash, note_name, note_text)
except Exception as exc: except Exception as exc:
debug_panel( debug_panel(
"add-file note write failed", "add-file note write failed",
[ [
("store", backend_name), ("store", backend_name),
("hash", resolved_hash), ("hash", resolved_hash),
("note", "sub"), ("note", note_name),
("error", exc), ("error", exc),
], ],
border_style="yellow", border_style="yellow",
) )
lyric_note = Add_File._get_note_text(result, pipe_obj, "lyric") _write_note("sub", Add_File._get_note_text(result, pipe_obj, "sub"))
if lyric_note and supports_note_association: _write_note("lyric", Add_File._get_note_text(result, pipe_obj, "lyric"))
try: _write_note("chapters", Add_File._get_note_text(result, pipe_obj, "chapters"))
setter = getattr(backend, "set_note", None) _write_note("caption", Add_File._get_note_text(result, pipe_obj, "caption"))
if callable(setter):
setter(resolved_hash, "lyric", lyric_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "lyric"),
("error", exc),
],
border_style="yellow",
)
chapters_note = Add_File._get_note_text(result, pipe_obj, "chapters")
if chapters_note and supports_note_association:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "chapters", chapters_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "chapters"),
("error", exc),
],
border_style="yellow",
)
caption_note = Add_File._get_note_text(result, pipe_obj, "caption")
if caption_note and supports_note_association:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "caption", caption_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "caption"),
("error", exc),
],
border_style="yellow",
)
meta: Dict[str, meta: Dict[str,
Any] = {} Any] = {}
try: try:
if not is_folder_backend: if not is_folder_backend:
_set_status("loading stored metadata")
meta = backend.get_metadata(resolved_hash) or {} meta = backend.get_metadata(resolved_hash) or {}
except Exception: except Exception:
meta = {} meta = {}
@@ -2886,9 +2857,11 @@ class Add_File(Cmdlet):
media_path, media_path,
delete_source=delete_after_effective delete_source=delete_after_effective
) )
_clear_status()
return 0 return 0
except Exception as exc: except Exception as exc:
_clear_status()
log( log(
f"❌ Failed to add file to backend '{backend_name}': {exc}", f"❌ Failed to add file to backend '{backend_name}': {exc}",
file=sys.stderr file=sys.stderr
@@ -2907,12 +2880,12 @@ class Add_File(Cmdlet):
List[str]]]], List[str]]]],
config: Dict[str, config: Dict[str,
Any], Any],
store_instance: Optional[Store] = None, store_instance: Optional[BackendRegistry] = None,
) -> None: ) -> None:
"""Apply deferred URL associations in bulk, grouped per backend.""" """Apply deferred URL associations in bulk, grouped per backend."""
try: try:
store = store_instance if store_instance is not None else Store(config) backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception: except Exception:
return return
@@ -2920,10 +2893,10 @@ class Add_File(Cmdlet):
if not pairs: if not pairs:
continue continue
try: try:
backend, store, _exc = sh.get_store_backend( backend, backend_registry, _exc = sh.get_store_backend(
config, config,
backend_name, backend_name,
store_registry=store, store_registry=backend_registry,
) )
if backend is None: if backend is None:
continue continue
@@ -2960,12 +2933,12 @@ class Add_File(Cmdlet):
List[str]]]], List[str]]]],
config: Dict[str, config: Dict[str,
Any], Any],
store_instance: Optional[Store] = None, store_instance: Optional[BackendRegistry] = None,
) -> None: ) -> None:
"""Apply deferred tag associations in bulk, grouped per backend.""" """Apply deferred tag associations in bulk, grouped per backend."""
try: try:
store = store_instance if store_instance is not None else Store(config) backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception: except Exception:
return return
@@ -2974,7 +2947,7 @@ class Add_File(Cmdlet):
pending or {}, pending or {},
bulk_method_name="add_tags_bulk", bulk_method_name="add_tags_bulk",
single_method_name="add_tag", single_method_name="add_tag",
store_registry=store, store_registry=backend_registry,
pass_config_to_bulk=False, pass_config_to_bulk=False,
pass_config_to_single=False, pass_config_to_single=False,
) )
cmdlet/file/delete.py
+3 -3
View File
@@ -9,7 +9,7 @@ from pathlib import Path
from SYS.logger import debug, log from SYS.logger import debug, log
from PluginCore.registry import get_plugin from PluginCore.registry import get_plugin
from Store import Store from PluginCore.backend_registry import BackendRegistry
from .. import _shared as sh from .. import _shared as sh
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.result_table_helpers import add_row_columns from SYS.result_table_helpers import add_row_columns
@@ -156,7 +156,7 @@ class Delete_File(sh.Cmdlet):
backend = None backend = None
try: try:
if instance: if instance:
registry = Store(config) registry = BackendRegistry(config)
if registry.is_available(str(store)): if registry.is_available(str(store)):
backend = registry[str(store)] backend = registry[str(store)]
except Exception: except Exception:
@@ -242,7 +242,7 @@ class Delete_File(sh.Cmdlet):
try: try:
# Re-use an already resolved backend when available. # Re-use an already resolved backend when available.
if backend is None: if backend is None:
registry = Store(config) registry = BackendRegistry(config)
if registry.is_available(str(store)): if registry.is_available(str(store)):
backend = registry[str(store)] backend = registry[str(store)]
cmdlet/file/download.py
+1 -1
View File
@@ -1661,7 +1661,7 @@ class Download_File(Cmdlet):
"""Initialize store registry and determine whether a Hydrus backend is usable.""" """Initialize store registry and determine whether a Hydrus backend is usable."""
storage = None storage = None
try: try:
from Store import Store as _Store from PluginCore.backend_registry import BackendRegistry as _Store
storage = _Store(config) storage = _Store(config)
except Exception: except Exception:
cmdlet/file/search.py
+12 -12
View File
@@ -1865,8 +1865,8 @@ class search_file(Cmdlet):
worker_id = str(uuid.uuid4()) worker_id = str(uuid.uuid4())
from Store import Store from PluginCore.backend_registry import BackendRegistry
storage_registry = Store(config=config or {}) storage_registry = BackendRegistry(config=config or {})
if not storage_registry.list_backends(): if not storage_registry.list_backends():
# Internal refreshes should not trigger config panels or stop progress. # Internal refreshes should not trigger config panels or stop progress.
@@ -1911,8 +1911,8 @@ class search_file(Cmdlet):
except Exception: except Exception:
pass pass
from Store.registry import list_configured_backend_names, get_backend_instance from PluginCore.backend_registry import list_configured_backend_names, get_backend_instance
from Store._base import Store as BaseStore from PluginCore.backend_base import BackendBase
backend_to_search = storage_backend or None backend_to_search = storage_backend or None
@@ -1929,13 +1929,13 @@ class search_file(Cmdlet):
for h in hash_query: for h in hash_query:
resolved_backend_name: Optional[str] = None resolved_backend_name: Optional[str] = None
resolved_backend = None resolved_backend = None
store_registry = None backend_registry_cache = None
for backend_name in backends_to_try: for backend_name in backends_to_try:
backend, store_registry, _exc = get_preferred_store_backend( backend, backend_registry_cache, _exc = get_preferred_store_backend(
config, config,
backend_name, backend_name,
store_registry=store_registry, store_registry=backend_registry_cache,
suppress_debug=True, suppress_debug=True,
) )
if backend is None: if backend is None:
@@ -2128,7 +2128,7 @@ class search_file(Cmdlet):
db.update_worker_status(worker_id, "error") db.update_worker_status(worker_id, "error")
return 1 return 1
if type(target_backend).search is BaseStore.search: if type(target_backend).search is BackendBase.search:
log( log(
f"Backend '{backend_to_search}' does not support searching", f"Backend '{backend_to_search}' does not support searching",
file=sys.stderr, file=sys.stderr,
@@ -2138,13 +2138,13 @@ class search_file(Cmdlet):
results = target_backend.search(query, limit=limit) results = target_backend.search(query, limit=limit)
else: else:
all_results = [] all_results = []
store_registry = None backend_registry_cache = None
for backend_name in list_configured_backend_names(config or {}): for backend_name in list_configured_backend_names(config or {}):
try: try:
backend, store_registry, _exc = get_preferred_store_backend( backend, backend_registry_cache, _exc = get_preferred_store_backend(
config, config,
backend_name, backend_name,
store_registry=store_registry, store_registry=backend_registry_cache,
suppress_debug=True, suppress_debug=True,
) )
if backend is None: if backend is None:
@@ -2154,7 +2154,7 @@ class search_file(Cmdlet):
searched_backends.append(backend_name) searched_backends.append(backend_name)
if type(backend).search is BaseStore.search: if type(backend).search is BackendBase.search:
continue continue
backend_results = backend.search( backend_results = backend.search(
cmdlet/file/trim.py
+3 -2
View File
@@ -14,6 +14,7 @@ from urllib.parse import urlparse
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.item_accessors import get_store_name from SYS.item_accessors import get_store_name
from SYS.utils import sha256_file from SYS.utils import sha256_file
from PluginCore.backend_registry import BackendRegistry
from .. import _shared as sh from .. import _shared as sh
Cmdlet = sh.Cmdlet Cmdlet = sh.Cmdlet
@@ -166,8 +167,8 @@ def _persist_alt_relationship(
) -> None: ) -> None:
"""Persist directional alt -> king relationship in the given backend.""" """Persist directional alt -> king relationship in the given backend."""
try: try:
store = Store(config) backend_registry = BackendRegistry(config)
backend: Any = store[str(store_name)] backend: Any = backend_registry[str(store_name)]
except Exception: except Exception:
return return
cmdlet/get_metadata.py
+66 -23
View File
@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Sequence, Optional from typing import Any, Dict, Sequence, Optional, List
import json import json
import sys import sys
@@ -235,6 +235,50 @@ class Get_Metadata(Cmdlet):
columns_to_add.append(("Duration(s)", "")) columns_to_add.append(("Duration(s)", ""))
add_row_columns(table, columns_to_add) add_row_columns(table, columns_to_add)
@staticmethod
def _extract_metadata_tags(metadata: Dict[str, Any]) -> List[str]:
tags: List[str] = []
def _append(tag_value: Any) -> None:
text = str(tag_value or "").strip()
if text and text not in tags:
tags.append(text)
def _walk_tag_values(value: Any) -> None:
if isinstance(value, str):
_append(value)
return
if isinstance(value, dict):
for nested_value in value.values():
_walk_tag_values(nested_value)
return
if isinstance(value, (list, tuple, set)):
for nested_value in value:
_walk_tag_values(nested_value)
raw_tags = metadata.get("tags")
if isinstance(raw_tags, dict):
for service_data in raw_tags.values():
if not isinstance(service_data, dict):
continue
matched_tag_key = False
for key, tag_mapping in service_data.items():
if "tag" not in str(key).strip().lower():
continue
matched_tag_key = True
_walk_tag_values(tag_mapping)
if not matched_tag_key:
_walk_tag_values(service_data)
elif isinstance(raw_tags, list):
for tag_value in raw_tags:
_append(tag_value)
for key in ("tags_flat", "tag"):
raw_value = metadata.get(key)
_walk_tag_values(raw_value)
return tags
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Execute get-metadata cmdlet - retrieve and display file metadata. """Execute get-metadata cmdlet - retrieve and display file metadata.
@@ -309,30 +353,29 @@ class Get_Metadata(Cmdlet):
else: else:
item_tags = [str(t) for t in item_tags] item_tags = [str(t) for t in item_tags]
# Extract tags from metadata response instead of making a separate get_tag() request metadata_tags = self._extract_metadata_tags(metadata)
# This prevents duplicate API calls to Hydrus (metadata already includes tags) if not metadata_tags:
metadata_tags = metadata.get("tags") get_tag = getattr(backend, "get_tag", None)
if isinstance(metadata_tags, dict): if callable(get_tag):
# metadata["tags"] is {service_key: {service_data}} try:
for service_data in metadata_tags.values(): backend_tags, _source = get_tag(file_hash, config=config)
if isinstance(service_data, dict): metadata_tags = [
display_tags = service_data.get("display_tags", {}) str(tag) for tag in (backend_tags or [])
if isinstance(display_tags, dict): if str(tag or "").strip()
# display_tags is typically {status: tag_list} ]
for tag_list in display_tags.values(): except Exception:
if isinstance(tag_list, list): metadata_tags = []
for t in tag_list:
ts = str(t) if t else "" for tag_value in metadata_tags:
if ts and ts not in item_tags: tag_text = str(tag_value or "").strip()
item_tags.append(ts) if not tag_text:
# Check for title tag continue
if not get_field(result, "title") and ts.lower().startswith("title:"): if tag_text not in item_tags:
parts = ts.split(":", 1) item_tags.append(tag_text)
if not get_field(result, "title") and tag_text.lower().startswith("title:"):
parts = tag_text.split(":", 1)
if len(parts) > 1: if len(parts) > 1:
title = parts[1].strip() title = parts[1].strip()
break # Only use first status level
if any(t for t in item_tags if str(t).lower().startswith("title:")):
break # Found title tag, stop searching services
# Extract metadata fields # Extract metadata fields
cmdlet/get_url.py
+3 -3
View File
@@ -22,7 +22,7 @@ from SYS.logger import log
from SYS.payload_builders import build_file_result_payload from SYS.payload_builders import build_file_result_payload
from SYS.result_publication import publish_result_table from SYS.result_publication import publish_result_table
from SYS.result_table import Table from SYS.result_table import Table
from Store import Store from PluginCore.backend_registry import BackendRegistry
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -250,7 +250,7 @@ class Get_Url(Cmdlet):
MAX_RESULTS = 256 MAX_RESULTS = 256
try: try:
storage = Store(config) storage = BackendRegistry(config)
store_names = storage.list_backends() if hasattr(storage, store_names = storage.list_backends() if hasattr(storage,
"list_backends") else [] "list_backends") else []
@@ -508,7 +508,7 @@ class Get_Url(Cmdlet):
# Get backend and retrieve url # Get backend and retrieve url
try: try:
storage = Store(config) storage = BackendRegistry(config)
backend = storage[store_name] backend = storage[store_name]
urls = backend.get_url(file_hash) urls = backend.get_url(file_hash)
cmdlet/metadata/tag_add.py
+8 -2
View File
@@ -13,7 +13,6 @@ from SYS.result_publication import publish_result_table
from SYS import models from SYS import models
from SYS import pipeline as ctx from SYS import pipeline as ctx
from .. import _shared as sh from .. import _shared as sh
from Store import Store # retained for test monkeypatch compatibility
normalize_result_input = sh.normalize_result_input normalize_result_input = sh.normalize_result_input
filter_results_by_temp = sh.filter_results_by_temp filter_results_by_temp = sh.filter_results_by_temp
@@ -31,6 +30,7 @@ should_show_help = sh.should_show_help
get_field = sh.get_field get_field = sh.get_field
_FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$") _FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$")
_DETAIL_PANEL_LIMIT = 9
def _normalize_title_for_extract(text: str) -> str: def _normalize_title_for_extract(text: str) -> str:
@@ -1191,7 +1191,13 @@ class Add_Tag(Cmdlet):
subject = display_items[0] if len(display_items) == 1 else list(display_items) subject = display_items[0] if len(display_items) == 1 else list(display_items)
# Use helper to display items and make them @-selectable # Use helper to display items and make them @-selectable
from ._shared import display_and_persist_items from ._shared import display_and_persist_items
display_and_persist_items(list(display_items), title="Result", subject=subject) display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom"
display_and_persist_items(
list(display_items),
title="Result",
subject=subject,
display_type=display_type,
)
except Exception: except Exception:
pass pass
cmdlet/metadata/tag_delete.py
+86 -2
View File
@@ -22,6 +22,9 @@ get_field = sh.get_field
from SYS.logger import debug, log from SYS.logger import debug, log
_DETAIL_PANEL_LIMIT = 9
def _matches_target( def _matches_target(
item: Any, item: Any,
target_hash: str | None, target_hash: str | None,
@@ -57,15 +60,56 @@ def _set_result_tags(result: Any, tags: list[str]) -> None:
normalized = list(tags or []) normalized = list(tags or [])
set_field(result, "tag", normalized) set_field(result, "tag", normalized)
def _update_tag_columns(columns: Any) -> Any:
if not isinstance(columns, (list, tuple)):
return columns
updated_columns = []
changed = False
tag_value = ", ".join(normalized)
for column in columns:
if isinstance(column, tuple) and len(column) == 2:
label, existing_value = column
if str(label).strip().lower() in {"tag", "tags"}:
updated_columns.append((label, tag_value))
changed = True
else:
updated_columns.append((label, existing_value))
elif isinstance(column, list) and len(column) == 2:
label, existing_value = column
if str(label).strip().lower() in {"tag", "tags"}:
updated_columns.append([label, tag_value])
changed = True
else:
updated_columns.append([label, existing_value])
elif isinstance(column, dict):
label = column.get("name") or column.get("label") or column.get("key")
if str(label).strip().lower() in {"tag", "tags"}:
updated_column = dict(column)
updated_column["value"] = tag_value
updated_columns.append(updated_column)
changed = True
else:
updated_columns.append(column)
else:
updated_columns.append(column)
if not changed:
return columns
if isinstance(columns, tuple):
return tuple(updated_columns)
return updated_columns
if isinstance(result, dict): if isinstance(result, dict):
result["tags_flat"] = list(normalized)
if "tags" in result: if "tags" in result:
result["tags"] = list(normalized) result["tags"] = list(normalized)
result["columns"] = _update_tag_columns(result.get("columns"))
for container_name in ("extra", "metadata", "full_metadata"): for container_name in ("extra", "metadata", "full_metadata"):
container = result.get(container_name) container = result.get(container_name)
if not isinstance(container, dict): if not isinstance(container, dict):
continue continue
if "tag" in container:
container["tag"] = list(normalized) container["tag"] = list(normalized)
container["tags_flat"] = list(normalized)
if "tags" in container: if "tags" in container:
container["tags"] = list(normalized) container["tags"] = list(normalized)
return return
@@ -74,12 +118,23 @@ def _set_result_tags(result: Any, tags: list[str]) -> None:
setattr(result, "tags", list(normalized)) setattr(result, "tags", list(normalized))
except Exception: except Exception:
pass pass
try:
setattr(result, "tags_flat", list(normalized))
except Exception:
pass
try:
columns = getattr(result, "columns", None)
updated_columns = _update_tag_columns(columns)
if updated_columns is not columns:
setattr(result, "columns", updated_columns)
except Exception:
pass
for container_name in ("extra", "metadata", "full_metadata"): for container_name in ("extra", "metadata", "full_metadata"):
container = getattr(result, container_name, None) container = getattr(result, container_name, None)
if not isinstance(container, dict): if not isinstance(container, dict):
continue continue
if "tag" in container:
container["tag"] = list(normalized) container["tag"] = list(normalized)
container["tags_flat"] = list(normalized)
if "tags" in container: if "tags" in container:
container["tags"] = list(normalized) container["tags"] = list(normalized)
@@ -514,6 +569,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Process each item # Process each item
success_count = 0 success_count = 0
stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx is None) or bool(
getattr(stage_ctx, "is_last_stage", False)
)
display_items: list[Any] = []
# If we have TagItems and no args, we are deleting the tags themselves # If we have TagItems and no args, we are deleting the tags themselves
# If we have Files (or other objects) and args, we are deleting tags FROM those files # If we have Files (or other objects) and args, we are deleting tags FROM those files
@@ -658,6 +718,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if title_value: if title_value:
_apply_title_to_result(item, title_value) _apply_title_to_result(item, title_value)
_refresh_result_table_tags(new_tags, h, store_str, path) _refresh_result_table_tags(new_tags, h, store_str, path)
if is_last_stage:
display_items.append(item)
try: try:
ctx.emit(item) ctx.emit(item)
except Exception: except Exception:
@@ -667,12 +729,34 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
for item, item_hash, item_path, store_str in items_needing_individual: for item, item_hash, item_path, store_str in items_needing_individual:
if _process_deletion(tags_arg, item_hash, item_path, store_str, config, result=item): if _process_deletion(tags_arg, item_hash, item_path, store_str, config, result=item):
success_count += 1 success_count += 1
if is_last_stage:
display_items.append(item)
try: try:
ctx.emit(item) ctx.emit(item)
except Exception: except Exception:
pass pass
if success_count > 0: if success_count > 0:
if is_last_stage and display_items:
try:
subject = display_items[0] if len(display_items) == 1 else list(display_items)
from ._shared import display_and_persist_items
display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom"
display_and_persist_items(
list(display_items),
title="Result",
subject=subject,
display_type=display_type,
)
except Exception:
pass
try:
if stage_ctx is not None:
stage_ctx.emits = []
except Exception:
pass
return 0 return 0
return 1 return 1
cmdlet/metadata/tag_get.py
+8 -8
View File
@@ -426,9 +426,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
backend = None backend = None
if is_store_backed: if is_store_backed:
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True) storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_name)] backend = storage[str(store_name)]
except Exception: except Exception:
backend = None backend = None
@@ -445,9 +445,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
store_for_scrape = get_field(result, "store", None) store_for_scrape = get_field(result, "store", None)
if file_hash_for_scrape and store_for_scrape: if file_hash_for_scrape and store_for_scrape:
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True) storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_for_scrape)] backend = storage[str(store_for_scrape)]
current_tags, _src = backend.get_tag(file_hash_for_scrape, config=config) current_tags, _src = backend.get_tag(file_hash_for_scrape, config=config)
if isinstance(current_tags, (list, tuple, set)) and current_tags: if isinstance(current_tags, (list, tuple, set)) and current_tags:
@@ -764,9 +764,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
) )
return 0 return 0
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True) storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_name)] backend = storage[str(store_name)]
ok = bool(backend.add_tag(file_hash, apply_tags, config=config)) ok = bool(backend.add_tag(file_hash, apply_tags, config=config))
if not ok: if not ok:
@@ -896,9 +896,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Get tags using storage backend # Get tags using storage backend
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True) storage = BackendRegistry(config, suppress_debug=True)
backend = storage[store_name] backend = storage[store_name]
current, source = backend.get_tag(file_hash, config=config) current, source = backend.get_tag(file_hash, config=config)
current = list(current or []) current = list(current or [])
cmdlet/metadata/url_add.py
+2 -2
View File
@@ -6,7 +6,7 @@ import sys
from SYS import pipeline as ctx from SYS import pipeline as ctx
from .. import _shared as sh from .. import _shared as sh
from SYS.logger import log from SYS.logger import log
from Store import Store from PluginCore.backend_registry import BackendRegistry
class Add_Url(sh.Cmdlet): class Add_Url(sh.Cmdlet):
@@ -117,7 +117,7 @@ class Add_Url(sh.Cmdlet):
# Get backend and add url # Get backend and add url
try: try:
storage = Store(config) storage = BackendRegistry(config)
# Build batches per store. # Build batches per store.
store_override = parsed.get("instance") store_override = parsed.get("instance")
plugins/README.md
+1 -1
View File
@@ -65,4 +65,4 @@ Bundled walkthrough:
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance <name>` uploads. - The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance <name>` uploads. - The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims. - The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its configured-backend adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `PluginCore.backend_registry`; the proxy exists only so generic backend callers can still target configured Hydrus instances.
plugins/fileio/__init__.py
+2 -2
View File
@@ -214,9 +214,9 @@ class FileIO(Provider):
try: try:
pipe_obj = kwargs.get("pipe_obj") pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None: if pipe_obj is not None:
from Store import Store from PluginCore.backend_registry import BackendRegistry
Store( BackendRegistry(
self.config, self.config,
suppress_debug=True suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj, ).try_add_url_for_pipe_object(pipe_obj,
plugins/internetarchive/__init__.py
+2 -2
View File
@@ -1296,9 +1296,9 @@ class InternetArchive(Provider):
try: try:
if pipe_obj is not None: if pipe_obj is not None:
from Store import Store from PluginCore.backend_registry import BackendRegistry
Store( BackendRegistry(
self.config, self.config,
suppress_debug=True suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj, ).try_add_url_for_pipe_object(pipe_obj,
plugins/matrix/__init__.py
+2 -2
View File
@@ -636,10 +636,10 @@ class Matrix(TablePluginMixin, Provider):
try: try:
pipe_obj = kwargs.get("pipe_obj") pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None: if pipe_obj is not None:
from Store import Store from PluginCore.backend_registry import BackendRegistry
# Prefer the direct media download URL for storage backends. # Prefer the direct media download URL for storage backends.
Store( BackendRegistry(
self.config, self.config,
suppress_debug=True suppress_debug=True
).try_add_url_for_pipe_object( ).try_add_url_for_pipe_object(
plugins/mpv/LUA/main.lua
+9 -9
View File
@@ -901,7 +901,7 @@ function M._load_store_choices_direct_async(cb)
'from SYS.logger import set_thread_stream', 'from SYS.logger import set_thread_stream',
'set_thread_stream(sys.stderr)', 'set_thread_stream(sys.stderr)',
'from SYS.config import load_config', 'from SYS.config import load_config',
'from Store.registry import list_configured_backend_names', 'from PluginCore.backend_registry import list_configured_backend_names',
'config = load_config()', 'config = load_config()',
'choices = list_configured_backend_names(config) or []', 'choices = list_configured_backend_names(config) or []',
'sys.stdout.write(json.dumps({"choices": choices}, ensure_ascii=False))', 'sys.stdout.write(json.dumps({"choices": choices}, ensure_ascii=False))',
@@ -2842,7 +2842,7 @@ local function _start_screenshot_store_save(store, out_path, tags)
if screenshot_url == '' or not screenshot_url:match('^https?://') then if screenshot_url == '' or not screenshot_url:match('^https?://') then
screenshot_url = '' screenshot_url = ''
end end
local cmd = 'file -add -store ' .. quote_pipeline_arg(store) local cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store)
.. ' -path ' .. quote_pipeline_arg(out_path) .. ' -path ' .. quote_pipeline_arg(out_path)
if screenshot_url ~= '' then if screenshot_url ~= '' then
cmd = cmd .. ' -url ' .. quote_pipeline_arg(screenshot_url) cmd = cmd .. ' -url ' .. quote_pipeline_arg(screenshot_url)
@@ -5864,7 +5864,7 @@ mp.register_script_message('medios-download-pick-store', function(json)
local pipeline_cmd = 'file -download -url ' .. quote_pipeline_arg(url) local pipeline_cmd = 'file -download -url ' .. quote_pipeline_arg(url)
.. ' -query ' .. quote_pipeline_arg(query) .. ' -query ' .. quote_pipeline_arg(query)
.. ' | file -add -store ' .. quote_pipeline_arg(store) .. ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store)
_set_selected_store(store) _set_selected_store(store)
_queue_pipeline_in_repl( _queue_pipeline_in_repl(
@@ -6331,16 +6331,16 @@ local function _start_trim_with_range(range)
pipeline_cmd = pipeline_cmd =
'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) .. 'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) ..
' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) ..
' | file -add -path ' .. quote_pipeline_arg(output_path) .. ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) ..
' -store "' .. selected_store .. '"' .. ' -path ' .. quote_pipeline_arg(output_path) ..
' | add-relationship -store "' .. selected_store .. '"' .. ' | add-relationship -store "' .. selected_store .. '"' ..
' -to-hash ' .. quote_pipeline_arg(store_hash.hash) ' -to-hash ' .. quote_pipeline_arg(store_hash.hash)
else else
pipeline_cmd = pipeline_cmd =
'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) .. 'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) ..
' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) ..
' | file -add -path ' .. quote_pipeline_arg(output_path) .. ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store_hash.store) ..
' -store "' .. store_hash.store .. '"' .. ' -path ' .. quote_pipeline_arg(output_path) ..
' | add-relationship -store "' .. store_hash.store .. '"' .. ' | add-relationship -store "' .. store_hash.store .. '"' ..
' -to-hash ' .. quote_pipeline_arg(store_hash.hash) ' -to-hash ' .. quote_pipeline_arg(store_hash.hash)
end end
@@ -6350,8 +6350,8 @@ local function _start_trim_with_range(range)
if selected_store then if selected_store then
_lua_log('trim: building file -add command to selected_store=' .. selected_store) _lua_log('trim: building file -add command to selected_store=' .. selected_store)
-- Don't add title if empty - the file path will be used as title by default -- Don't add title if empty - the file path will be used as title by default
pipeline_cmd = 'file -add -path ' .. quote_pipeline_arg(output_path) .. pipeline_cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) ..
' -store "' .. selected_store .. '"' ' -path ' .. quote_pipeline_arg(output_path)
_lua_log('trim: pipeline_cmd=' .. pipeline_cmd) _lua_log('trim: pipeline_cmd=' .. pipeline_cmd)
else else
mp.osd_message('Trim complete: ' .. output_path, 5) mp.osd_message('Trim complete: ' .. output_path, 5)
plugins/mpv/commands.py
+12 -14
View File
@@ -816,7 +816,7 @@ def _prefetch_notes_async(
set_notes_prefetch_pending, set_notes_prefetch_pending,
store_cached_notes, store_cached_notes,
) )
from Store import Store from PluginCore.backend_registry import BackendRegistry
cached = load_cached_notes(store, file_hash, config=cfg) cached = load_cached_notes(store, file_hash, config=cfg)
if cached is not None: if cached is not None:
@@ -824,7 +824,7 @@ def _prefetch_notes_async(
set_notes_prefetch_pending(store, file_hash, True) set_notes_prefetch_pending(store, file_hash, True)
registry = Store(cfg, suppress_debug=True) registry = BackendRegistry(cfg, suppress_debug=True)
if not registry.is_available(str(store)): if not registry.is_available(str(store)):
return return
backend = registry[str(store)] backend = registry[str(store)]
@@ -1674,14 +1674,14 @@ def _queue_items(
except Exception: except Exception:
hydrus_url = None hydrus_url = None
# Initialize Store registry for path resolution # Initialize backend registry for path resolution
file_storage = None file_storage = None
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
file_storage = Store(config or {}) file_storage = BackendRegistry(config or {})
except Exception as e: except Exception as e:
debug(f"Warning: Could not initialize Store registry: {e}", file=sys.stderr) debug(f"Warning: Could not initialize backend registry: {e}", file=sys.stderr)
_schedule_notes_prefetch(items, config) _schedule_notes_prefetch(items, config)
@@ -1831,12 +1831,10 @@ def _queue_items(
continue continue
new_targets.add(norm_key) new_targets.add(norm_key)
# Use memory:// M3U hack to pass title to MPV. # Use memory:// M3U to preserve titles in MPV's playlist UI.
# Avoid this for probable ytdl URLs because it can prevent the hook from triggering. # Avoid this only for probable ytdl URLs because it can prevent the hook from triggering.
safe_title = title.replace("\n", " ").replace("\r", "") if title else None safe_title = title.replace("\n", " ").replace("\r", "") if title else None
if title and hydrus_target: if title and not _is_probable_ytdl_url(target):
target_to_send = target
elif title and not _is_probable_ytdl_url(target):
# Sanitize title for M3U (remove newlines) # Sanitize title for M3U (remove newlines)
# Carry the store name for hash URLs so MPV.lyric can resolve the backend. # Carry the store name for hash URLs so MPV.lyric can resolve the backend.
# This is especially important for local file-server URLs like /get_files/file?hash=... # This is especially important for local file-server URLs like /get_files/file?hash=...
@@ -2587,12 +2585,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if file_storage is None: if file_storage is None:
try: try:
from Store import Store from PluginCore.backend_registry import BackendRegistry
file_storage = Store(config) file_storage = BackendRegistry(config)
except Exception as e: except Exception as e:
debug( debug(
f"Warning: Could not initialize Store registry: {e}", f"Warning: Could not initialize backend registry: {e}",
file=sys.stderr file=sys.stderr
) )
plugins/mpv/lyric.py
+12 -12
View File
@@ -1156,10 +1156,10 @@ def _infer_hydrus_store_from_url_target(*, target: str, config: dict) -> Optiona
return None return None
try: try:
from Store import Store as StoreRegistry from PluginCore.backend_registry import BackendRegistry
reg = StoreRegistry(config, suppress_debug=True) backend_registry = BackendRegistry(config, suppress_debug=True)
backends = [(name, reg[name]) for name in reg.list_backends()] backends = [(name, backend_registry[name]) for name in backend_registry.list_backends()]
except Exception: except Exception:
return None return None
@@ -1218,10 +1218,10 @@ def _resolve_store_backend_for_target(
return None, None return None, None
try: try:
from Store import Store as StoreRegistry from PluginCore.backend_registry import BackendRegistry
reg = StoreRegistry(config, suppress_debug=True) backend_registry = BackendRegistry(config, suppress_debug=True)
backend_names = list(reg.list_backends()) backend_names = list(backend_registry.list_backends())
except Exception: except Exception:
return None, None return None, None
@@ -1229,7 +1229,7 @@ def _resolve_store_backend_for_target(
for name in backend_names: for name in backend_names:
try: try:
backend = reg[name] backend = backend_registry[name]
except Exception: except Exception:
continue continue
@@ -1363,16 +1363,16 @@ def run_auto_overlay(
# Import the Store registry once so each track change doesn't re-import the module. # Import the Store registry once so each track change doesn't re-import the module.
try: try:
from Store import Store as _StoreRegistry # noqa: PLC0415 from PluginCore.backend_registry import BackendRegistry # noqa: PLC0415
_store_cls: Any = _StoreRegistry _backend_registry_cls: Any = BackendRegistry
except Exception: except Exception:
_store_cls = None _backend_registry_cls = None
def _make_registry() -> Optional[Any]: def _make_registry() -> Optional[Any]:
if _store_cls is None: if _backend_registry_cls is None:
return None return None
try: try:
return _store_cls(cfg, suppress_debug=True) return _backend_registry_cls(cfg, suppress_debug=True)
except Exception: except Exception:
return None return None
plugins/mpv/mpv_ipc.py
+1 -1
View File
@@ -472,7 +472,7 @@ class MPV:
pipeline = f"file -download -url {_q(url)} -query {_q(f'format:{fmt}')}" pipeline = f"file -download -url {_q(url)} -query {_q(f'format:{fmt}')}"
if store: if store:
pipeline += f" | file -add -instance {_q(store)}" pipeline += f" | file -add -plugin hydrusnetwork -instance {_q(store)}"
else: else:
pipeline += f" | file -add -plugin local -instance {_q(path or '')}" pipeline += f" | file -add -plugin local -instance {_q(path or '')}"
plugins/mpv/pipeline_helper.py
+3 -3
View File
@@ -133,7 +133,7 @@ def _store_choices_payload(choices: Any) -> Optional[str]:
def _load_store_choices_from_config(*, force_reload: bool = False) -> list[str]: def _load_store_choices_from_config(*, force_reload: bool = False) -> list[str]:
from Store.registry import list_configured_backend_names # noqa: WPS433 from PluginCore.backend_registry import list_configured_backend_names # noqa: WPS433
cfg = reload_config() if force_reload else load_config() cfg = reload_config() if force_reload else load_config()
return _normalize_store_choices(list_configured_backend_names(cfg or {})) return _normalize_store_choices(list_configured_backend_names(cfg or {}))
@@ -810,10 +810,10 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]:
"find-url", "find-url",
"find_url"}: "find_url"}:
try: try:
from Store import Store # noqa: WPS433 from PluginCore.backend_registry import BackendRegistry # noqa: WPS433
cfg = load_config() or {} cfg = load_config() or {}
storage = Store(config=cfg, suppress_debug=True) storage = BackendRegistry(config=cfg, suppress_debug=True)
raw_needles: list[str] = [] raw_needles: list[str] = []
if isinstance(data, dict): if isinstance(data, dict):
plugins/zeroxzero/__init__.py
+2 -2
View File
@@ -48,9 +48,9 @@ class ZeroXZero(Provider):
try: try:
pipe_obj = kwargs.get("pipe_obj") pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None: if pipe_obj is not None:
from Store import Store from PluginCore.backend_registry import BackendRegistry
Store( BackendRegistry(
self.config, self.config,
suppress_debug=True suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj, ).try_add_url_for_pipe_object(pipe_obj,
tool/ytdlp.py
+9 -10
View File
@@ -2620,6 +2620,7 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, config: Optional[Dict[str, Any]] = None) -> Any: def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, config: Optional[Dict[str, Any]] = None) -> Any:
from contextvars import copy_context
import threading import threading
from typing import cast from typing import cast
@@ -2631,17 +2632,18 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co
except Exception as exc: except Exception as exc:
result_container[1] = exc result_container[1] = exc
# Preserve the active pipeline ContextVar state so yt-dlp progress hooks in the
# worker thread can still see the shared Live UI and stage context.
download_context = copy_context()
# Use daemon=True so a hung download doesn't block process exit if the wall timeout hits. # Use daemon=True so a hung download doesn't block process exit if the wall timeout hits.
thread = threading.Thread(target=_do_download, daemon=True) thread = threading.Thread(target=lambda: download_context.run(_do_download), daemon=True)
thread.start() thread.start()
start_time = time.monotonic() start_time = time.monotonic()
# We use two timeouts: # Keep only an activity timeout here. yt-dlp downloads can be legitimately slow
# 1. Activity timeout (no progress updates for X seconds) # for large media or constrained connections, and a fixed wall-clock cutoff can
# 2. Hard wall-clock timeout (total time for this URL) # abort healthy downloads even while progress is still arriving.
# The wall-clock timeout is slightly larger than the activity timeout
# to allow for slow-but-steady progress, up to a hard cap (e.g. 10 minutes).
wall_timeout = max(timeout_seconds * 2, 600)
_record_progress_activity(start_time) _record_progress_activity(start_time)
try: try:
@@ -2659,9 +2661,6 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co
if now - last_activity > timeout_seconds: if now - last_activity > timeout_seconds:
raise DownloadError(f"Download activity timeout after {timeout_seconds} seconds for {opts.url}") raise DownloadError(f"Download activity timeout after {timeout_seconds} seconds for {opts.url}")
# Check hard wall-clock timeout
if now - start_time > wall_timeout:
raise DownloadError(f"Download hard timeout after {wall_timeout} seconds for {opts.url}")
finally: finally:
_clear_progress_activity() _clear_progress_activity()