goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updated old legacy store names

Browse Source
This commit is contained in:
Nose
2026-05-23 13:49:47 -07:00
parent e8913d1344
commit a8cdd09e1e
31 changed files with 466 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Store/_base.py → PluginCore/backend_base.py
+16 -16
View File
@@ -1,6 +1,7 @@
"""Store backend base types.
"""Plugin-backed configured backend base types.
Concrete store implementations live in the `Store/` package.
Concrete storage backends live with their owning plugins and are resolved
through PluginCore.backend_registry.
"""
from __future__ import annotations
@@ -10,12 +11,12 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
class Store(ABC):
class BackendBase(ABC):
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
"""Return configuration schema for this store.
"""Return configuration schema for this backend.
Returns a list of dicts:
{
"key": "PATH",
@@ -32,27 +33,27 @@ class Store(ABC):
@property
def is_remote(self) -> bool:
"""True if the store is a remote service (e.g. Hydrus) rather than local disk."""
"""True if this backend is a remote service rather than local disk."""
return False
@property
def prefer_defer_tags(self) -> bool:
"""True if the store prefers tags to be applied after the file is added."""
"""True if this backend prefers tags to be applied after add_file()."""
return False
@property
def supports_url_association(self) -> bool:
"""True when this store supports associating URLs to files."""
"""True when this backend supports associating URLs to files."""
return False
@property
def supports_note_association(self) -> bool:
"""True when this store supports per-file named notes."""
"""True when this backend supports per-file named notes."""
return False
@property
def supports_relationship_association(self) -> bool:
"""True when this store supports file relationship links (king/alt/related)."""
"""True when this backend supports file relationship links."""
return False
@abstractmethod
@@ -111,9 +112,8 @@ class Store(ABC):
def delete_url_bulk(
self,
items: List[Tuple[str,
List[str]]],
**kwargs: Any
items: List[Tuple[str, List[str]]],
**kwargs: Any,
) -> bool:
"""Optional bulk url deletion.
@@ -162,7 +162,7 @@ class Store(ABC):
file_identifier: str,
name: str,
text: str,
**kwargs: Any
**kwargs: Any,
) -> bool:
"""Add or replace a named note for a file."""
raise NotImplementedError
@@ -173,7 +173,7 @@ class Store(ABC):
*,
ctx: Any,
stage_is_last: bool = True,
**_kwargs: Any
**_kwargs: Any,
) -> bool:
"""Optional hook for handling `@N` selection semantics.
@@ -187,4 +187,4 @@ class Store(ABC):
@abstractmethod
def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool:
"""Delete a named note for a file."""
raise NotImplementedError
raise NotImplementedError
Store/registry.py → PluginCore/backend_registry.py
+113 -213
View File
@@ -1,38 +1,31 @@
"""Store registry.
"""Configured plugin-backed backend registry.
Concrete store implementations live in the `Store/` package.
This module is the single source of truth for store discovery.
This registry is config-driven:
- Each store subtype (e.g. `hydrusnetwork`) maps to a concrete store class.
- Each store class advertises its required config keys via `StoreClass.__new__.keys`.
- Instances are created from config using those keys (case-insensitive lookup).
Backends are discovered from their owning plugins and instantiated from config.
"""
from __future__ import annotations
import importlib
import inspect
import pkgutil
import re
from typing import Any, Dict, Optional, Type
from SYS.logger import debug
from SYS.utils import expand_path
from Store._base import Store as BaseStore
from PluginCore.backend_base import BackendBase
_SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$")
_DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None
_PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BaseStore]]] = {}
_PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BackendBase]]] = {}
# Backends that failed to initialize earlier in the current process.
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>.
# Keyed by (backend_type, instance_key) where instance_key is the configured name
# under config.store.<type>.<instance_key>.
_FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {}
def _normalize_store_type(value: str) -> str:
def _normalize_backend_type(value: str) -> str:
return "".join(ch.lower() for ch in str(value or "") if ch.isalnum())
@@ -44,59 +37,23 @@ def _get_case_insensitive(mapping: Dict[str, Any], key: str) -> Any:
if key in mapping:
return mapping[key]
desired = _normalize_config_key(key)
for k, v in mapping.items():
if _normalize_config_key(k) == desired:
return v
for current_key, value in mapping.items():
if _normalize_config_key(current_key) == desired:
return value
return None
def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
"""Discover store classes from the Store package.
Convention:
- The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork).
"""
global _DISCOVERED_CLASSES_CACHE
if _DISCOVERED_CLASSES_CACHE is not None:
return _DISCOVERED_CLASSES_CACHE
import Store as store_pkg
discovered: Dict[str, Type[BaseStore]] = {}
for module_info in pkgutil.iter_modules(store_pkg.__path__):
module_name = module_info.name
if module_name.startswith(("_", "registry")):
continue
try:
module = importlib.import_module(f"Store.{module_name}")
for _, obj in vars(module).items():
if not inspect.isclass(obj):
continue
if obj is BaseStore:
continue
if not issubclass(obj, BaseStore):
continue
discovered[_normalize_store_type(obj.__name__)] = obj
except Exception as exc:
debug(f"[Store] Failed to import module '{module_name}': {exc}")
continue
_DISCOVERED_CLASSES_CACHE = discovered
return discovered
def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]:
discovered: Dict[str, Type[BaseStore]] = {}
def _extract_backend_classes(owner: Any) -> Dict[str, Type[BackendBase]]:
discovered: Dict[str, Type[BackendBase]] = {}
def _add_candidate(key: Any, candidate: Any) -> None:
if not inspect.isclass(candidate):
return
if candidate is BaseStore:
if candidate is BackendBase:
return
if not issubclass(candidate, BaseStore):
if not issubclass(candidate, BackendBase):
return
normalized = _normalize_store_type(str(key or candidate.__name__))
normalized = _normalize_backend_type(str(key or candidate.__name__))
if normalized:
discovered[normalized] = candidate
@@ -125,8 +82,8 @@ def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]:
return discovered
def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]:
normalized = _normalize_store_type(store_type)
def _discover_plugin_backend_class(backend_type: str) -> Optional[Type[BackendBase]]:
normalized = _normalize_backend_type(backend_type)
if not normalized:
return None
@@ -140,19 +97,19 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]:
_PLUGIN_DISCOVERED_CLASSES_CACHE[normalized] = None
return None
discovered: Dict[str, Type[BaseStore]] = {}
discovered: Dict[str, Type[BackendBase]] = {}
backend_hook = getattr(plugin_module, "get_store_backend_classes", None)
if callable(backend_hook):
try:
discovered.update(_extract_store_classes(backend_hook()))
discovered.update(_extract_backend_classes(backend_hook()))
except Exception as exc:
debug(f"[Store] Failed to load plugin store backends for '{normalized}': {exc}")
debug(f"[BackendRegistry] Failed to load plugin backends for '{normalized}': {exc}")
discovered.update(_extract_store_classes(getattr(plugin_module, "STORE_BACKENDS", None)))
discovered.update(_extract_backend_classes(getattr(plugin_module, "STORE_BACKENDS", None)))
if normalized not in discovered:
discovered.update(_extract_store_classes(plugin_module))
discovered.update(_extract_backend_classes(plugin_module))
resolved = discovered.get(normalized)
if resolved is None and len(discovered) == 1:
@@ -162,37 +119,24 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]:
return resolved
def _resolve_store_class(
store_type: str,
classes_by_type: Optional[Dict[str, Type[BaseStore]]] = None,
) -> Optional[Type[BaseStore]]:
normalized = _normalize_store_type(store_type)
def _resolve_backend_class(backend_type: str) -> Optional[Type[BackendBase]]:
normalized = _normalize_backend_type(backend_type)
if not normalized:
return None
plugin_resolved = _discover_plugin_store_class(normalized)
if plugin_resolved is not None:
return plugin_resolved
discovered = classes_by_type if classes_by_type is not None else _discover_store_classes()
resolved = discovered.get(normalized)
if resolved is not None:
return resolved
return None
return _discover_plugin_backend_class(normalized)
def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
# Support new config_schema() schema
if hasattr(store_cls, "config_schema") and callable(store_cls.config_schema):
def _required_keys_for(backend_cls: Type[BackendBase]) -> list[str]:
if hasattr(backend_cls, "config_schema") and callable(backend_cls.config_schema):
try:
schema = store_cls.config_schema()
schema = backend_cls.config_schema()
keys = []
if isinstance(schema, list):
for field in schema:
if isinstance(field, dict) and field.get("required"):
k = field.get("key")
if k:
keys.append(str(k))
key = field.get("key")
if key:
keys.append(str(key))
if keys:
return keys
except Exception:
@@ -200,21 +144,14 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
return []
# Store type names that have been converted to providers-only.
# These should be silently skipped without warning.
_PROVIDER_ONLY_STORE_NAMES = frozenset(("debrid", "alldebrid"))
_PROVIDER_ONLY_BACKEND_NAMES = frozenset(("debrid", "alldebrid"))
def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]:
if isinstance(instance_config, dict):
cfg_dict = dict(instance_config)
else:
cfg_dict = {}
def _build_kwargs(backend_cls: Type[BackendBase], instance_name: str, instance_config: Any) -> Dict[str, Any]:
cfg_dict = dict(instance_config) if isinstance(instance_config, dict) else {}
required = _required_keys_for(store_cls)
# If NAME is required but not present, allow the instance key to provide it.
if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None):
required = _required_keys_for(backend_cls)
if any(_normalize_config_key(key) == "NAME" for key in required) and _get_case_insensitive(cfg_dict, "NAME") is None:
cfg_dict["NAME"] = str(instance_name)
kwargs: Dict[str, Any] = {}
@@ -228,28 +165,24 @@ def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_confi
if missing:
raise ValueError(
f"Missing required keys for {store_cls.__name__}: {', '.join(missing)}"
f"Missing required keys for {backend_cls.__name__}: {', '.join(missing)}"
)
return kwargs
class Store:
class BackendRegistry:
def __init__(
self,
config: Optional[Dict[str,
Any]] = None,
suppress_debug: bool = False
config: Optional[Dict[str, Any]] = None,
suppress_debug: bool = False,
) -> None:
self._config = config or {}
self._suppress_debug = suppress_debug
self._backends: Dict[str,
BaseStore] = {}
self._backend_errors: Dict[str,
str] = {}
self._backend_types: Dict[str,
str] = {}
self._backends: Dict[str, BackendBase] = {}
self._backend_errors: Dict[str, str] = {}
self._backend_types: Dict[str, str] = {}
self._load_backends()
def _load_backends(self) -> None:
@@ -258,61 +191,54 @@ class Store:
store_cfg = {}
self._backend_types = {}
classes_by_type = _discover_store_classes()
for raw_store_type, instances in store_cfg.items():
for raw_backend_type, instances in store_cfg.items():
if not isinstance(instances, dict):
continue
store_type = _normalize_store_type(str(raw_store_type))
if store_type == "folder":
backend_type = _normalize_backend_type(str(raw_backend_type))
if backend_type == "folder":
continue
store_cls = _resolve_store_class(store_type, classes_by_type)
if store_cls is None:
# Skip provider-only names without debug warning
if store_type not in _PROVIDER_ONLY_STORE_NAMES and not self._suppress_debug:
debug(f"[Store] Unknown store type '{raw_store_type}'")
backend_cls = _resolve_backend_class(backend_type)
if backend_cls is None:
if backend_type not in _PROVIDER_ONLY_BACKEND_NAMES and not self._suppress_debug:
debug(f"[BackendRegistry] Unknown backend type '{raw_backend_type}'")
continue
for instance_name, instance_config in instances.items():
backend_name = str(instance_name)
# If this backend already failed earlier in this process, skip re-instantiation.
cache_key = (store_type, str(instance_name))
cache_key = (backend_type, str(instance_name))
cached_error = _FAILED_BACKEND_CACHE.get(cache_key)
if cached_error:
self._backend_errors[str(instance_name)] = str(cached_error)
if isinstance(instance_config, dict):
override_name = _get_case_insensitive(
dict(instance_config),
"NAME"
)
override_name = _get_case_insensitive(dict(instance_config), "NAME")
if override_name:
self._backend_errors[str(override_name)] = str(cached_error)
continue
try:
kwargs = _build_kwargs(
store_cls,
backend_cls,
str(instance_name),
instance_config
instance_config,
)
# Convenience normalization for filesystem-like paths.
for key in list(kwargs.keys()):
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
kwargs[key] = str(expand_path(kwargs[key]))
backend = store_cls(**kwargs)
backend = backend_cls(**kwargs)
backend_name = str(kwargs.get("NAME") or instance_name)
self._backends[backend_name] = backend
self._backend_types[backend_name] = store_type
self._backend_types[backend_name] = backend_type
except Exception as exc:
err_text = str(exc)
self._backend_errors[str(instance_name)] = err_text
_FAILED_BACKEND_CACHE[cache_key] = err_text
if not self._suppress_debug:
debug(
f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}"
f"[BackendRegistry] Failed to register {backend_cls.__name__} instance '{instance_name}': {exc}"
)
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
@@ -320,46 +246,41 @@ class Store:
if requested in self._backends:
return requested, None
requested_norm = _normalize_store_type(requested)
requested_norm = _normalize_backend_type(requested)
ci_matches = [
name for name in self._backends
if _normalize_store_type(name) == requested_norm
if _normalize_backend_type(name) == requested_norm
]
if len(ci_matches) == 1:
return ci_matches[0], None
if len(ci_matches) > 1:
return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}"
return None, f"Ambiguous backend alias '{backend_name}' matches {ci_matches}"
type_matches = [
name for name, store_type in self._backend_types.items()
if store_type == requested_norm
name for name, backend_type in self._backend_types.items()
if backend_type == requested_norm
]
if len(type_matches) == 1:
return type_matches[0], None
if len(type_matches) > 1:
return None, (
f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}"
f"Ambiguous backend alias '{backend_name}' matches type '{requested_norm}': {type_matches}"
)
prefix_matches = [
name for name, store_type in self._backend_types.items()
if store_type.startswith(requested_norm)
name for name, backend_type in self._backend_types.items()
if backend_type.startswith(requested_norm)
]
if len(prefix_matches) == 1:
return prefix_matches[0], None
if len(prefix_matches) > 1:
return None, (
f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}"
f"Ambiguous backend alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}"
)
return None, None
# get_backend_instance implementation moved to the bottom of this file to avoid
# instantiating all backends during startup (see function `get_backend_instance`).
# Duplicate _resolve_backend_name removed — the method is defined once earlier in the class.
def get_backend_error(self, backend_name: str) -> Optional[str]:
return self._backend_errors.get(str(backend_name))
@@ -367,19 +288,17 @@ class Store:
return sorted(self._backends.keys())
def list_searchable_backends(self) -> list[str]:
# De-duplicate backends by instance (aliases can point at the same object).
def _rank(name: str) -> int:
n = str(name or "").strip().lower()
if n == "temp":
normalized_name = str(name or "").strip().lower()
if normalized_name == "temp":
return 0
if n == "default":
if normalized_name == "default":
return 2
return 1
chosen: Dict[int,
str] = {}
chosen: Dict[int, str] = {}
for name, backend in self._backends.items():
if type(backend).search is BaseStore.search:
if type(backend).search is BackendBase.search:
continue
key = id(backend)
prev = chosen.get(key)
@@ -387,27 +306,20 @@ class Store:
chosen[key] = name
return sorted(chosen.values())
def __getitem__(self, backend_name: str) -> BaseStore:
def __getitem__(self, backend_name: str) -> BackendBase:
resolved, err = self._resolve_backend_name(backend_name)
if resolved:
return self._backends[resolved]
if err:
raise KeyError(
f"Unknown store backend: {backend_name}. {err}"
)
raise KeyError(
f"Unknown store backend: {backend_name}. Available: {list(self._backends.keys())}"
)
raise KeyError(f"Unknown backend: {backend_name}. {err}")
raise KeyError(f"Unknown backend: {backend_name}. Available: {list(self._backends.keys())}")
def is_available(self, backend_name: str) -> bool:
resolved, _err = self._resolve_backend_name(backend_name)
return resolved is not None
def try_add_url_for_pipe_object(self, pipe_obj: Any, url: str) -> bool:
"""Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that store backend.
Intended for providers to attach newly generated/hosted URLs back to an existing stored file.
"""
"""Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that backend."""
try:
url_text = str(url or "").strip()
if not url_text:
@@ -442,41 +354,28 @@ class Store:
def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]:
"""Return backend instance names present in the provided config WITHOUT instantiating backends.
This is a lightweight helper for CLI usage where we only need to know if a
configured backend exists (e.g., to distinguish a store name from a filesystem path)
without triggering backend initialization (which may perform network calls).
Behaviour:
- Only includes store types that map to a discovered save backend class.
- Skips folder/provider-only/unknown entries so UI pickers do not surface
non-ingest destinations such as provider helpers.
- For each configured store type, returns the per-instance NAME override
(case-insensitive) when present, otherwise the instance key.
"""
"""Return configured backend instance names without instantiating backends."""
try:
classes_by_type = _discover_store_classes()
names: list[str] = []
for section_name in ("store", "plugin", "provider"):
section_cfg = (config or {}).get(section_name) or {}
if not isinstance(section_cfg, dict):
continue
for raw_store_type, instances in section_cfg.items():
for raw_backend_type, instances in section_cfg.items():
if not isinstance(instances, dict):
continue
store_type = _normalize_store_type(str(raw_store_type))
if store_type == "folder" or store_type in _PROVIDER_ONLY_STORE_NAMES:
backend_type = _normalize_backend_type(str(raw_backend_type))
if backend_type == "folder" or backend_type in _PROVIDER_ONLY_BACKEND_NAMES:
continue
store_cls = _resolve_store_class(store_type, classes_by_type)
if store_cls is None:
backend_cls = _resolve_backend_class(backend_type)
if backend_cls is None:
continue
for instance_name, instance_config in instances.items():
try:
_build_kwargs(store_cls, str(instance_name), instance_config)
_build_kwargs(backend_cls, str(instance_name), instance_config)
except Exception:
continue
if isinstance(instance_config, dict):
@@ -493,18 +392,15 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]
return []
def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]:
"""Instantiate and return a single store backend by configured name.
This avoids creating all configured backends (and opening their DBs)
when only a single backend is needed (common in `get-file`/`get-metadata`).
The function first tries a lightweight match against raw config NAME/value to
avoid calling `_build_kwargs` (which can raise if keys are missing).
Returns None when no matching backend is found or instantiation fails.
"""
def get_backend_instance(
config: Optional[Dict[str, Any]],
backend_name: str,
*,
suppress_debug: bool = False,
) -> Optional[BackendBase]:
"""Instantiate and return a single configured backend by name."""
if not backend_name:
return None
classes_by_type = _discover_store_classes()
desired = str(backend_name or "").strip().lower()
for section_name in ("store", "plugin", "provider"):
@@ -512,29 +408,28 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
if not isinstance(section_cfg, dict):
continue
for raw_store_type, instances in section_cfg.items():
for raw_backend_type, instances in section_cfg.items():
if not isinstance(instances, dict):
continue
store_type = _normalize_store_type(str(raw_store_type))
store_cls = _resolve_store_class(store_type, classes_by_type)
if store_cls is None:
backend_type = _normalize_backend_type(str(raw_backend_type))
backend_cls = _resolve_backend_class(backend_type)
if backend_cls is None:
continue
# Fast path: match using raw 'NAME' or 'name' in config without building full kwargs
for instance_name, instance_cfg in instances.items():
candidate_alias = None
if isinstance(instance_cfg, dict):
candidate_alias = (
instance_cfg.get("NAME") or instance_cfg.get("name")
)
candidate_alias = instance_cfg.get("NAME") or instance_cfg.get("name")
candidate_alias = str(candidate_alias or instance_name).strip()
if candidate_alias.lower() != desired:
continue
try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg)
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}/{section_name}): {exc}")
debug(
f"[BackendRegistry] Can't build kwargs for '{instance_name}' ({backend_type}/{section_name}): {exc}"
)
return None
try:
for key in list(kwargs.keys()):
@@ -543,17 +438,15 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
except Exception:
pass
try:
backend = store_cls(**kwargs)
return backend
return backend_cls(**kwargs)
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}")
debug(f"[BackendRegistry] Failed to instantiate backend '{candidate_alias}': {exc}")
return None
# Fallback: build kwargs for each instance and compare resolved NAME
for instance_name, instance_cfg in instances.items():
try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg)
except Exception:
continue
alias = str(kwargs.get("NAME") or instance_name).strip()
@@ -566,13 +459,20 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *,
except Exception:
pass
try:
backend = store_cls(**kwargs)
return backend
return backend_cls(**kwargs)
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{alias}': {exc}")
debug(f"[BackendRegistry] Failed to instantiate backend '{alias}': {exc}")
return None
if not suppress_debug:
debug(f"[Store] Backend '{backend_name}' not found in config")
debug(f"[BackendRegistry] Backend '{backend_name}' not found in config")
return None
__all__ = [
"BackendBase",
"BackendRegistry",
"get_backend_instance",
"list_configured_backend_names",
]
PluginCore/base.py
+1 -1
View File
@@ -621,7 +621,7 @@ class Provider(ABC):
raise NotImplementedError(f"Plugin '{self.name}' does not support upload")
# -----------------------------------------------------------------------
# Storage interface — mirrors Store._base.Store.
# Storage interface — mirrors PluginCore.backend_base.BackendBase.
# Plugins that act as file repositories override these methods.
# All raise NotImplementedError by default; override selectively.
# -----------------------------------------------------------------------
SYS/cmdlet_spec.py
+2 -2
View File
@@ -125,9 +125,9 @@ class SharedArgs:
return
try:
from Store.registry import Store as StoreRegistry
from PluginCore.backend_registry import BackendRegistry
registry = StoreRegistry(config=config, suppress_debug=True)
registry = BackendRegistry(config=config, suppress_debug=True)
available = registry.list_backends()
if available:
SharedArgs._cached_available_stores = available
SYS/pipeline.py
+6 -6
View File
@@ -1730,21 +1730,21 @@ class PipelineExecutor:
if store_keys:
try:
from Store.registry import Store as StoreRegistry
from PluginCore.backend_registry import BackendRegistry
store_registry = StoreRegistry(config, suppress_debug=True)
_backend_names = list(store_registry.list_backends() or [])
backend_registry = BackendRegistry(config, suppress_debug=True)
_backend_names = list(backend_registry.list_backends() or [])
_backend_by_lower = {
str(n).lower(): str(n)
for n in _backend_names if str(n).strip()
}
for name in store_keys:
resolved_name = name
if not store_registry.is_available(resolved_name):
if not backend_registry.is_available(resolved_name):
resolved_name = _backend_by_lower.get(str(name).lower(), name)
if not store_registry.is_available(resolved_name):
if not backend_registry.is_available(resolved_name):
continue
backend = store_registry[resolved_name]
backend = backend_registry[resolved_name]
selector = getattr(backend, "selector", None)
if selector is None:
continue
Store/__init__.py
-7
View File
@@ -1,7 +0,0 @@
from Store._base import Store as BaseStore
from Store.registry import Store
__all__ = [
"Store",
"BaseStore",
]
TUI.py
+5 -5
View File
@@ -53,7 +53,7 @@ from SYS.result_table import Table, extract_hash_value, extract_store_value, get
from SYS.config import load_config # type: ignore # noqa: E402
from SYS.database import db
from Store.registry import Store as StoreRegistry # type: ignore # noqa: E402
from PluginCore.backend_registry import BackendRegistry # type: ignore # noqa: E402
from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402
from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402
@@ -261,7 +261,7 @@ class TagEditorPopup(ModalScreen[None]):
if not store_key or not hash_value:
return None
try:
registry = StoreRegistry(config=cfg, suppress_debug=True)
registry = BackendRegistry(config=cfg, suppress_debug=True)
except Exception:
return []
match = None
@@ -984,9 +984,9 @@ class PipelineHubApp(App):
stores: List[str] = []
try:
stores = StoreRegistry(config=cfg, suppress_debug=True).list_backends()
stores = BackendRegistry(config=cfg, suppress_debug=True).list_backends()
except Exception:
logger.exception("Failed to list store backends from StoreRegistry")
logger.exception("Failed to list store backends from BackendRegistry")
stores = []
# Always offer a reasonable default even if config is missing.
@@ -1380,7 +1380,7 @@ class PipelineHubApp(App):
cfg = {}
try:
registry = StoreRegistry(config=cfg, suppress_debug=True)
registry = BackendRegistry(config=cfg, suppress_debug=True)
except Exception:
return None
cmdlet/_shared.py
+3 -3
View File
@@ -1084,9 +1084,9 @@ def get_store_backend(
registry = store_registry
if registry is None:
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
registry = Store(config or {}, suppress_debug=suppress_debug)
registry = BackendRegistry(config or {}, suppress_debug=suppress_debug)
except Exception as exc:
return None, None, exc
@@ -1110,7 +1110,7 @@ def get_preferred_store_backend(
"""Prefer a targeted backend instance before falling back to registry lookup."""
direct_exc: Optional[Exception] = None
try:
from Store.registry import get_backend_instance
from PluginCore.backend_registry import get_backend_instance
backend = get_backend_instance(
config or {},
cmdlet/delete_url.py
+2 -2
View File
@@ -14,7 +14,7 @@ from ._shared import (
normalize_hash,
)
from SYS.logger import log
from Store import Store
from PluginCore.backend_registry import BackendRegistry
class Delete_Url(Cmdlet):
@@ -106,7 +106,7 @@ class Delete_Url(Cmdlet):
# Get backend and delete url
try:
storage = Store(config)
storage = BackendRegistry(config)
store_override = parsed.get("instance")
cmdlet/file/add.py
+67 -94
View File
@@ -16,7 +16,7 @@ from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store
from PluginCore.backend_registry import BackendRegistry
from API.HTTP import _download_direct_file
from .. import _shared as sh
@@ -45,21 +45,21 @@ SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS
class _CommandDependencies:
"""Command-scope cache for Store and plugin instances to avoid repeated instantiation."""
"""Command-scope cache for the backend registry and plugin instances."""
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
self._store: Optional[Store] = None
self._backend_registry: Optional[BackendRegistry] = None
self._plugins: Dict[str, Any] = {}
def get_store(self) -> Optional[Store]:
"""Lazily initialize and return the command-scope Store instance."""
if self._store is None:
def get_backend_registry(self) -> Optional[BackendRegistry]:
"""Lazily initialize and return the command-scope backend registry."""
if self._backend_registry is None:
try:
self._store = Store(self.config)
self._backend_registry = BackendRegistry(self.config)
except Exception:
self._store = None
return self._store
self._backend_registry = None
return self._backend_registry
def get_plugin(self, name: str) -> Optional[Any]:
"""Cached plugin lookup by name."""
@@ -236,7 +236,7 @@ class Add_File(Cmdlet):
# Initialize command-scope dependency context (caches Store/plugins)
deps = _CommandDependencies(config)
storage_registry = deps.get_store()
storage_registry = deps.get_backend_registry()
path_arg = parsed.get("path")
location = parsed.get("instance")
@@ -354,8 +354,8 @@ class Add_File(Cmdlet):
is_storage_backend_location = False
if location:
try:
store_for_lookup = storage_registry or deps.get_store()
is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None
backend_registry_for_lookup = storage_registry or deps.get_backend_registry()
is_storage_backend_location = Add_File._resolve_backend_by_name(backend_registry_for_lookup, str(location)) is not None
except Exception:
is_storage_backend_location = False
@@ -697,8 +697,8 @@ class Add_File(Cmdlet):
if location:
try:
store = storage_registry or deps.get_store()
resolved_backend = Add_File._resolve_backend_by_name(store, str(location))
backend_registry = storage_registry or deps.get_backend_registry()
resolved_backend = Add_File._resolve_backend_by_name(backend_registry, str(location))
if resolved_backend is not None:
code = self._handle_storage_backend(
item,
@@ -845,7 +845,7 @@ class Add_File(Cmdlet):
hash_values: List[str],
config: Dict[str,
Any],
store_instance: Optional[Store] = None,
store_instance: Optional[BackendRegistry] = None,
) -> Optional[List[Any]]:
"""Run search-file for a list of hashes and promote the table to a display overlay.
@@ -1053,7 +1053,7 @@ class Add_File(Cmdlet):
str]]],
config: Dict[str,
Any],
store_instance: Optional[Store] = None,
store_instance: Optional[BackendRegistry] = None,
deps: Optional[_CommandDependencies] = None,
) -> None:
"""Persist relationships to backends that support relationships.
@@ -1067,7 +1067,7 @@ class Add_File(Cmdlet):
deps = _CommandDependencies(config)
try:
store = store_instance if store_instance is not None else deps.get_store()
backend_registry = store_instance if store_instance is not None else deps.get_backend_registry()
except Exception:
return
@@ -1076,7 +1076,7 @@ class Add_File(Cmdlet):
continue
try:
backend = store[str(backend_name)]
backend = backend_registry[str(backend_name)]
except Exception:
continue
@@ -1356,9 +1356,9 @@ class Add_File(Cmdlet):
try:
if deps is None:
deps = _CommandDependencies(config)
store = store_instance or deps.get_store()
backend = Add_File._resolve_backend_by_name(store, r_store)
backend_registry = store_instance or deps.get_backend_registry()
backend = Add_File._resolve_backend_by_name(backend_registry, r_store)
if backend is not None:
mp = backend.get_file(r_hash)
if isinstance(mp, Path) and mp.exists():
@@ -1497,14 +1497,14 @@ class Add_File(Cmdlet):
explicit_instance = str(instance_name or "").strip() or None
try:
storage = store_instance if store_instance is not None else Store(config)
backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception:
storage = None
backend_registry = None
try:
resolved_name, backend = resolver(
explicit_instance,
storage=storage,
storage=backend_registry,
require_explicit=bool(explicit_instance),
)
except TypeError:
@@ -1622,8 +1622,8 @@ class Add_File(Cmdlet):
if deps is None:
deps = _CommandDependencies(config)
store = store_instance or deps.get_store()
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None
backend_registry = store_instance or deps.get_backend_registry()
backend = Add_File._resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None
if backend is None:
return None, None, None
@@ -2475,10 +2475,23 @@ class Add_File(Cmdlet):
List[str]]]]] = None,
suppress_last_stage_overlay: bool = False,
auto_search_file: bool = True,
store_instance: Optional[Store] = None,
store_instance: Optional[BackendRegistry] = None,
) -> int:
"""Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.)."""
##log(f"Adding file to storage backend '{backend_name}': {media_path.name}", file=sys.stderr)
pipeline_progress = PipelineProgress(ctx)
def _set_status(text: str) -> None:
try:
pipeline_progress.set_status(f"{backend_name}: {text}")
except Exception:
pass
def _clear_status() -> None:
try:
pipeline_progress.clear_status()
except Exception:
pass
delete_after_effective = bool(delete_after)
# ... (lines omitted for brevity but I need to keep them contextually correct)
@@ -2510,11 +2523,11 @@ class Add_File(Cmdlet):
pass
try:
store = store_instance if store_instance is not None else Store(config)
backend, store, backend_exc = sh.get_preferred_store_backend(
backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
backend, backend_registry, backend_exc = sh.get_preferred_store_backend(
config,
backend_name,
store_registry=store,
store_registry=backend_registry,
suppress_debug=True,
)
if backend is None:
@@ -2654,6 +2667,8 @@ class Add_File(Cmdlet):
tag=upload_tags,
url=[] if ((defer_url_association and url) or (not supports_url_association)) else url,
file_hash=f_hash,
pipeline_progress=pipeline_progress,
transfer_label=title or media_path.name,
)
##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr)
@@ -2704,6 +2719,7 @@ class Add_File(Cmdlet):
try:
adder = getattr(backend, "add_tag", None)
if callable(adder):
_set_status("applying deferred tags")
adder(resolved_hash, list(tags))
except Exception as exc:
log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr)
@@ -2724,88 +2740,43 @@ class Add_File(Cmdlet):
try:
# Folder.add_file already persists URLs, avoid extra DB traffic here.
if not is_folder_backend:
_set_status("associating urls")
backend.add_url(resolved_hash, list(url))
except Exception:
pass
# If a subtitle note was provided upstream (e.g., download-media writes notes.sub),
# persist it automatically like add-note would.
sub_note = Add_File._get_note_text(result, pipe_obj, "sub")
if sub_note and supports_note_association:
def _write_note(note_name: str, note_text: Optional[str]) -> None:
if not note_text or not supports_note_association:
return
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "sub", sub_note)
_set_status(f"writing {note_name} note")
setter(resolved_hash, note_name, note_text)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "sub"),
("note", note_name),
("error", exc),
],
border_style="yellow",
)
lyric_note = Add_File._get_note_text(result, pipe_obj, "lyric")
if lyric_note and supports_note_association:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "lyric", lyric_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "lyric"),
("error", exc),
],
border_style="yellow",
)
chapters_note = Add_File._get_note_text(result, pipe_obj, "chapters")
if chapters_note and supports_note_association:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "chapters", chapters_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "chapters"),
("error", exc),
],
border_style="yellow",
)
caption_note = Add_File._get_note_text(result, pipe_obj, "caption")
if caption_note and supports_note_association:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
setter(resolved_hash, "caption", caption_note)
except Exception as exc:
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "caption"),
("error", exc),
],
border_style="yellow",
)
_write_note("sub", Add_File._get_note_text(result, pipe_obj, "sub"))
_write_note("lyric", Add_File._get_note_text(result, pipe_obj, "lyric"))
_write_note("chapters", Add_File._get_note_text(result, pipe_obj, "chapters"))
_write_note("caption", Add_File._get_note_text(result, pipe_obj, "caption"))
meta: Dict[str,
Any] = {}
try:
if not is_folder_backend:
_set_status("loading stored metadata")
meta = backend.get_metadata(resolved_hash) or {}
except Exception:
meta = {}
@@ -2886,9 +2857,11 @@ class Add_File(Cmdlet):
media_path,
delete_source=delete_after_effective
)
_clear_status()
return 0
except Exception as exc:
_clear_status()
log(
f"❌ Failed to add file to backend '{backend_name}': {exc}",
file=sys.stderr
@@ -2907,12 +2880,12 @@ class Add_File(Cmdlet):
List[str]]]],
config: Dict[str,
Any],
store_instance: Optional[Store] = None,
store_instance: Optional[BackendRegistry] = None,
) -> None:
"""Apply deferred URL associations in bulk, grouped per backend."""
try:
store = store_instance if store_instance is not None else Store(config)
backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception:
return
@@ -2920,10 +2893,10 @@ class Add_File(Cmdlet):
if not pairs:
continue
try:
backend, store, _exc = sh.get_store_backend(
backend, backend_registry, _exc = sh.get_store_backend(
config,
backend_name,
store_registry=store,
store_registry=backend_registry,
)
if backend is None:
continue
@@ -2960,12 +2933,12 @@ class Add_File(Cmdlet):
List[str]]]],
config: Dict[str,
Any],
store_instance: Optional[Store] = None,
store_instance: Optional[BackendRegistry] = None,
) -> None:
"""Apply deferred tag associations in bulk, grouped per backend."""
try:
store = store_instance if store_instance is not None else Store(config)
backend_registry = store_instance if store_instance is not None else BackendRegistry(config)
except Exception:
return
@@ -2974,7 +2947,7 @@ class Add_File(Cmdlet):
pending or {},
bulk_method_name="add_tags_bulk",
single_method_name="add_tag",
store_registry=store,
store_registry=backend_registry,
pass_config_to_bulk=False,
pass_config_to_single=False,
)
cmdlet/file/delete.py
+3 -3
View File
@@ -9,7 +9,7 @@ from pathlib import Path
from SYS.logger import debug, log
from PluginCore.registry import get_plugin
from Store import Store
from PluginCore.backend_registry import BackendRegistry
from .. import _shared as sh
from SYS import pipeline as ctx
from SYS.result_table_helpers import add_row_columns
@@ -156,7 +156,7 @@ class Delete_File(sh.Cmdlet):
backend = None
try:
if instance:
registry = Store(config)
registry = BackendRegistry(config)
if registry.is_available(str(store)):
backend = registry[str(store)]
except Exception:
@@ -242,7 +242,7 @@ class Delete_File(sh.Cmdlet):
try:
# Re-use an already resolved backend when available.
if backend is None:
registry = Store(config)
registry = BackendRegistry(config)
if registry.is_available(str(store)):
backend = registry[str(store)]
cmdlet/file/download.py
+1 -1
View File
@@ -1661,7 +1661,7 @@ class Download_File(Cmdlet):
"""Initialize store registry and determine whether a Hydrus backend is usable."""
storage = None
try:
from Store import Store as _Store
from PluginCore.backend_registry import BackendRegistry as _Store
storage = _Store(config)
except Exception:
cmdlet/file/search.py
+12 -12
View File
@@ -1865,8 +1865,8 @@ class search_file(Cmdlet):
worker_id = str(uuid.uuid4())
from Store import Store
storage_registry = Store(config=config or {})
from PluginCore.backend_registry import BackendRegistry
storage_registry = BackendRegistry(config=config or {})
if not storage_registry.list_backends():
# Internal refreshes should not trigger config panels or stop progress.
@@ -1911,8 +1911,8 @@ class search_file(Cmdlet):
except Exception:
pass
from Store.registry import list_configured_backend_names, get_backend_instance
from Store._base import Store as BaseStore
from PluginCore.backend_registry import list_configured_backend_names, get_backend_instance
from PluginCore.backend_base import BackendBase
backend_to_search = storage_backend or None
@@ -1929,13 +1929,13 @@ class search_file(Cmdlet):
for h in hash_query:
resolved_backend_name: Optional[str] = None
resolved_backend = None
store_registry = None
backend_registry_cache = None
for backend_name in backends_to_try:
backend, store_registry, _exc = get_preferred_store_backend(
backend, backend_registry_cache, _exc = get_preferred_store_backend(
config,
backend_name,
store_registry=store_registry,
store_registry=backend_registry_cache,
suppress_debug=True,
)
if backend is None:
@@ -2128,7 +2128,7 @@ class search_file(Cmdlet):
db.update_worker_status(worker_id, "error")
return 1
if type(target_backend).search is BaseStore.search:
if type(target_backend).search is BackendBase.search:
log(
f"Backend '{backend_to_search}' does not support searching",
file=sys.stderr,
@@ -2138,13 +2138,13 @@ class search_file(Cmdlet):
results = target_backend.search(query, limit=limit)
else:
all_results = []
store_registry = None
backend_registry_cache = None
for backend_name in list_configured_backend_names(config or {}):
try:
backend, store_registry, _exc = get_preferred_store_backend(
backend, backend_registry_cache, _exc = get_preferred_store_backend(
config,
backend_name,
store_registry=store_registry,
store_registry=backend_registry_cache,
suppress_debug=True,
)
if backend is None:
@@ -2154,7 +2154,7 @@ class search_file(Cmdlet):
searched_backends.append(backend_name)
if type(backend).search is BaseStore.search:
if type(backend).search is BackendBase.search:
continue
backend_results = backend.search(
cmdlet/file/trim.py
+3 -2
View File
@@ -14,6 +14,7 @@ from urllib.parse import urlparse
from SYS.logger import log, debug
from SYS.item_accessors import get_store_name
from SYS.utils import sha256_file
from PluginCore.backend_registry import BackendRegistry
from .. import _shared as sh
Cmdlet = sh.Cmdlet
@@ -166,8 +167,8 @@ def _persist_alt_relationship(
) -> None:
"""Persist directional alt -> king relationship in the given backend."""
try:
store = Store(config)
backend: Any = store[str(store_name)]
backend_registry = BackendRegistry(config)
backend: Any = backend_registry[str(store_name)]
except Exception:
return
cmdlet/get_metadata.py
+68 -25
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Dict, Sequence, Optional
from typing import Any, Dict, Sequence, Optional, List
import json
import sys
@@ -235,6 +235,50 @@ class Get_Metadata(Cmdlet):
columns_to_add.append(("Duration(s)", ""))
add_row_columns(table, columns_to_add)
@staticmethod
def _extract_metadata_tags(metadata: Dict[str, Any]) -> List[str]:
tags: List[str] = []
def _append(tag_value: Any) -> None:
text = str(tag_value or "").strip()
if text and text not in tags:
tags.append(text)
def _walk_tag_values(value: Any) -> None:
if isinstance(value, str):
_append(value)
return
if isinstance(value, dict):
for nested_value in value.values():
_walk_tag_values(nested_value)
return
if isinstance(value, (list, tuple, set)):
for nested_value in value:
_walk_tag_values(nested_value)
raw_tags = metadata.get("tags")
if isinstance(raw_tags, dict):
for service_data in raw_tags.values():
if not isinstance(service_data, dict):
continue
matched_tag_key = False
for key, tag_mapping in service_data.items():
if "tag" not in str(key).strip().lower():
continue
matched_tag_key = True
_walk_tag_values(tag_mapping)
if not matched_tag_key:
_walk_tag_values(service_data)
elif isinstance(raw_tags, list):
for tag_value in raw_tags:
_append(tag_value)
for key in ("tags_flat", "tag"):
raw_value = metadata.get(key)
_walk_tag_values(raw_value)
return tags
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Execute get-metadata cmdlet - retrieve and display file metadata.
@@ -309,30 +353,29 @@ class Get_Metadata(Cmdlet):
else:
item_tags = [str(t) for t in item_tags]
# Extract tags from metadata response instead of making a separate get_tag() request
# This prevents duplicate API calls to Hydrus (metadata already includes tags)
metadata_tags = metadata.get("tags")
if isinstance(metadata_tags, dict):
# metadata["tags"] is {service_key: {service_data}}
for service_data in metadata_tags.values():
if isinstance(service_data, dict):
display_tags = service_data.get("display_tags", {})
if isinstance(display_tags, dict):
# display_tags is typically {status: tag_list}
for tag_list in display_tags.values():
if isinstance(tag_list, list):
for t in tag_list:
ts = str(t) if t else ""
if ts and ts not in item_tags:
item_tags.append(ts)
# Check for title tag
if not get_field(result, "title") and ts.lower().startswith("title:"):
parts = ts.split(":", 1)
if len(parts) > 1:
title = parts[1].strip()
break # Only use first status level
if any(t for t in item_tags if str(t).lower().startswith("title:")):
break # Found title tag, stop searching services
metadata_tags = self._extract_metadata_tags(metadata)
if not metadata_tags:
get_tag = getattr(backend, "get_tag", None)
if callable(get_tag):
try:
backend_tags, _source = get_tag(file_hash, config=config)
metadata_tags = [
str(tag) for tag in (backend_tags or [])
if str(tag or "").strip()
]
except Exception:
metadata_tags = []
for tag_value in metadata_tags:
tag_text = str(tag_value or "").strip()
if not tag_text:
continue
if tag_text not in item_tags:
item_tags.append(tag_text)
if not get_field(result, "title") and tag_text.lower().startswith("title:"):
parts = tag_text.split(":", 1)
if len(parts) > 1:
title = parts[1].strip()
# Extract metadata fields
cmdlet/get_url.py
+3 -3
View File
@@ -22,7 +22,7 @@ from SYS.logger import log
from SYS.payload_builders import build_file_result_payload
from SYS.result_publication import publish_result_table
from SYS.result_table import Table
from Store import Store
from PluginCore.backend_registry import BackendRegistry
from SYS import pipeline as ctx
@@ -250,7 +250,7 @@ class Get_Url(Cmdlet):
MAX_RESULTS = 256
try:
storage = Store(config)
storage = BackendRegistry(config)
store_names = storage.list_backends() if hasattr(storage,
"list_backends") else []
@@ -508,7 +508,7 @@ class Get_Url(Cmdlet):
# Get backend and retrieve url
try:
storage = Store(config)
storage = BackendRegistry(config)
backend = storage[store_name]
urls = backend.get_url(file_hash)
cmdlet/metadata/tag_add.py
+8 -2
View File
@@ -13,7 +13,6 @@ from SYS.result_publication import publish_result_table
from SYS import models
from SYS import pipeline as ctx
from .. import _shared as sh
from Store import Store # retained for test monkeypatch compatibility
normalize_result_input = sh.normalize_result_input
filter_results_by_temp = sh.filter_results_by_temp
@@ -31,6 +30,7 @@ should_show_help = sh.should_show_help
get_field = sh.get_field
_FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$")
_DETAIL_PANEL_LIMIT = 9
def _normalize_title_for_extract(text: str) -> str:
@@ -1191,7 +1191,13 @@ class Add_Tag(Cmdlet):
subject = display_items[0] if len(display_items) == 1 else list(display_items)
# Use helper to display items and make them @-selectable
from ._shared import display_and_persist_items
display_and_persist_items(list(display_items), title="Result", subject=subject)
display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom"
display_and_persist_items(
list(display_items),
title="Result",
subject=subject,
display_type=display_type,
)
except Exception:
pass
cmdlet/metadata/tag_delete.py
+88 -4
View File
@@ -22,6 +22,9 @@ get_field = sh.get_field
from SYS.logger import debug, log
_DETAIL_PANEL_LIMIT = 9
def _matches_target(
item: Any,
target_hash: str | None,
@@ -57,15 +60,56 @@ def _set_result_tags(result: Any, tags: list[str]) -> None:
normalized = list(tags or [])
set_field(result, "tag", normalized)
def _update_tag_columns(columns: Any) -> Any:
if not isinstance(columns, (list, tuple)):
return columns
updated_columns = []
changed = False
tag_value = ", ".join(normalized)
for column in columns:
if isinstance(column, tuple) and len(column) == 2:
label, existing_value = column
if str(label).strip().lower() in {"tag", "tags"}:
updated_columns.append((label, tag_value))
changed = True
else:
updated_columns.append((label, existing_value))
elif isinstance(column, list) and len(column) == 2:
label, existing_value = column
if str(label).strip().lower() in {"tag", "tags"}:
updated_columns.append([label, tag_value])
changed = True
else:
updated_columns.append([label, existing_value])
elif isinstance(column, dict):
label = column.get("name") or column.get("label") or column.get("key")
if str(label).strip().lower() in {"tag", "tags"}:
updated_column = dict(column)
updated_column["value"] = tag_value
updated_columns.append(updated_column)
changed = True
else:
updated_columns.append(column)
else:
updated_columns.append(column)
if not changed:
return columns
if isinstance(columns, tuple):
return tuple(updated_columns)
return updated_columns
if isinstance(result, dict):
result["tags_flat"] = list(normalized)
if "tags" in result:
result["tags"] = list(normalized)
result["columns"] = _update_tag_columns(result.get("columns"))
for container_name in ("extra", "metadata", "full_metadata"):
container = result.get(container_name)
if not isinstance(container, dict):
continue
if "tag" in container:
container["tag"] = list(normalized)
container["tag"] = list(normalized)
container["tags_flat"] = list(normalized)
if "tags" in container:
container["tags"] = list(normalized)
return
@@ -74,12 +118,23 @@ def _set_result_tags(result: Any, tags: list[str]) -> None:
setattr(result, "tags", list(normalized))
except Exception:
pass
try:
setattr(result, "tags_flat", list(normalized))
except Exception:
pass
try:
columns = getattr(result, "columns", None)
updated_columns = _update_tag_columns(columns)
if updated_columns is not columns:
setattr(result, "columns", updated_columns)
except Exception:
pass
for container_name in ("extra", "metadata", "full_metadata"):
container = getattr(result, container_name, None)
if not isinstance(container, dict):
continue
if "tag" in container:
container["tag"] = list(normalized)
container["tag"] = list(normalized)
container["tags_flat"] = list(normalized)
if "tags" in container:
container["tags"] = list(normalized)
@@ -514,6 +569,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Process each item
success_count = 0
stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx is None) or bool(
getattr(stage_ctx, "is_last_stage", False)
)
display_items: list[Any] = []
# If we have TagItems and no args, we are deleting the tags themselves
# If we have Files (or other objects) and args, we are deleting tags FROM those files
@@ -658,6 +718,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if title_value:
_apply_title_to_result(item, title_value)
_refresh_result_table_tags(new_tags, h, store_str, path)
if is_last_stage:
display_items.append(item)
try:
ctx.emit(item)
except Exception:
@@ -667,12 +729,34 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
for item, item_hash, item_path, store_str in items_needing_individual:
if _process_deletion(tags_arg, item_hash, item_path, store_str, config, result=item):
success_count += 1
if is_last_stage:
display_items.append(item)
try:
ctx.emit(item)
except Exception:
pass
if success_count > 0:
if is_last_stage and display_items:
try:
subject = display_items[0] if len(display_items) == 1 else list(display_items)
from ._shared import display_and_persist_items
display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom"
display_and_persist_items(
list(display_items),
title="Result",
subject=subject,
display_type=display_type,
)
except Exception:
pass
try:
if stage_ctx is not None:
stage_ctx.emits = []
except Exception:
pass
return 0
return 1
cmdlet/metadata/tag_get.py
+8 -8
View File
@@ -426,9 +426,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
backend = None
if is_store_backed:
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True)
storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_name)]
except Exception:
backend = None
@@ -445,9 +445,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
store_for_scrape = get_field(result, "store", None)
if file_hash_for_scrape and store_for_scrape:
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True)
storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_for_scrape)]
current_tags, _src = backend.get_tag(file_hash_for_scrape, config=config)
if isinstance(current_tags, (list, tuple, set)) and current_tags:
@@ -764,9 +764,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
)
return 0
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True)
storage = BackendRegistry(config, suppress_debug=True)
backend = storage[str(store_name)]
ok = bool(backend.add_tag(file_hash, apply_tags, config=config))
if not ok:
@@ -896,9 +896,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Get tags using storage backend
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
storage = Store(config, suppress_debug=True)
storage = BackendRegistry(config, suppress_debug=True)
backend = storage[store_name]
current, source = backend.get_tag(file_hash, config=config)
current = list(current or [])
cmdlet/metadata/url_add.py
+2 -2
View File
@@ -6,7 +6,7 @@ import sys
from SYS import pipeline as ctx
from .. import _shared as sh
from SYS.logger import log
from Store import Store
from PluginCore.backend_registry import BackendRegistry
class Add_Url(sh.Cmdlet):
@@ -117,7 +117,7 @@ class Add_Url(sh.Cmdlet):
# Get backend and add url
try:
storage = Store(config)
storage = BackendRegistry(config)
# Build batches per store.
store_override = parsed.get("instance")
plugins/README.md
+1 -1
View File
@@ -65,4 +65,4 @@ Bundled walkthrough:
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims.
- The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its configured-backend adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `PluginCore.backend_registry`; the proxy exists only so generic backend callers can still target configured Hydrus instances.
plugins/fileio/__init__.py
+2 -2
View File
@@ -214,9 +214,9 @@ class FileIO(Provider):
try:
pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
Store(
BackendRegistry(
self.config,
suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj,
plugins/internetarchive/__init__.py
+2 -2
View File
@@ -1296,9 +1296,9 @@ class InternetArchive(Provider):
try:
if pipe_obj is not None:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
Store(
BackendRegistry(
self.config,
suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj,
plugins/matrix/__init__.py
+2 -2
View File
@@ -636,10 +636,10 @@ class Matrix(TablePluginMixin, Provider):
try:
pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
# Prefer the direct media download URL for storage backends.
Store(
BackendRegistry(
self.config,
suppress_debug=True
).try_add_url_for_pipe_object(
plugins/mpv/LUA/main.lua
+9 -9
View File
@@ -901,7 +901,7 @@ function M._load_store_choices_direct_async(cb)
'from SYS.logger import set_thread_stream',
'set_thread_stream(sys.stderr)',
'from SYS.config import load_config',
'from Store.registry import list_configured_backend_names',
'from PluginCore.backend_registry import list_configured_backend_names',
'config = load_config()',
'choices = list_configured_backend_names(config) or []',
'sys.stdout.write(json.dumps({"choices": choices}, ensure_ascii=False))',
@@ -2842,7 +2842,7 @@ local function _start_screenshot_store_save(store, out_path, tags)
if screenshot_url == '' or not screenshot_url:match('^https?://') then
screenshot_url = ''
end
local cmd = 'file -add -store ' .. quote_pipeline_arg(store)
local cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store)
.. ' -path ' .. quote_pipeline_arg(out_path)
if screenshot_url ~= '' then
cmd = cmd .. ' -url ' .. quote_pipeline_arg(screenshot_url)
@@ -5864,7 +5864,7 @@ mp.register_script_message('medios-download-pick-store', function(json)
local pipeline_cmd = 'file -download -url ' .. quote_pipeline_arg(url)
.. ' -query ' .. quote_pipeline_arg(query)
.. ' | file -add -store ' .. quote_pipeline_arg(store)
.. ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store)
_set_selected_store(store)
_queue_pipeline_in_repl(
@@ -6331,16 +6331,16 @@ local function _start_trim_with_range(range)
pipeline_cmd =
'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) ..
' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) ..
' | file -add -path ' .. quote_pipeline_arg(output_path) ..
' -store "' .. selected_store .. '"' ..
' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) ..
' -path ' .. quote_pipeline_arg(output_path) ..
' | add-relationship -store "' .. selected_store .. '"' ..
' -to-hash ' .. quote_pipeline_arg(store_hash.hash)
else
pipeline_cmd =
'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) ..
' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) ..
' | file -add -path ' .. quote_pipeline_arg(output_path) ..
' -store "' .. store_hash.store .. '"' ..
' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store_hash.store) ..
' -path ' .. quote_pipeline_arg(output_path) ..
' | add-relationship -store "' .. store_hash.store .. '"' ..
' -to-hash ' .. quote_pipeline_arg(store_hash.hash)
end
@@ -6350,8 +6350,8 @@ local function _start_trim_with_range(range)
if selected_store then
_lua_log('trim: building file -add command to selected_store=' .. selected_store)
-- Don't add title if empty - the file path will be used as title by default
pipeline_cmd = 'file -add -path ' .. quote_pipeline_arg(output_path) ..
' -store "' .. selected_store .. '"'
pipeline_cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) ..
' -path ' .. quote_pipeline_arg(output_path)
_lua_log('trim: pipeline_cmd=' .. pipeline_cmd)
else
mp.osd_message('Trim complete: ' .. output_path, 5)
plugins/mpv/commands.py
+12 -14
View File
@@ -816,7 +816,7 @@ def _prefetch_notes_async(
set_notes_prefetch_pending,
store_cached_notes,
)
from Store import Store
from PluginCore.backend_registry import BackendRegistry
cached = load_cached_notes(store, file_hash, config=cfg)
if cached is not None:
@@ -824,7 +824,7 @@ def _prefetch_notes_async(
set_notes_prefetch_pending(store, file_hash, True)
registry = Store(cfg, suppress_debug=True)
registry = BackendRegistry(cfg, suppress_debug=True)
if not registry.is_available(str(store)):
return
backend = registry[str(store)]
@@ -1674,14 +1674,14 @@ def _queue_items(
except Exception:
hydrus_url = None
# Initialize Store registry for path resolution
# Initialize backend registry for path resolution
file_storage = None
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
file_storage = Store(config or {})
file_storage = BackendRegistry(config or {})
except Exception as e:
debug(f"Warning: Could not initialize Store registry: {e}", file=sys.stderr)
debug(f"Warning: Could not initialize backend registry: {e}", file=sys.stderr)
_schedule_notes_prefetch(items, config)
@@ -1831,12 +1831,10 @@ def _queue_items(
continue
new_targets.add(norm_key)
# Use memory:// M3U hack to pass title to MPV.
# Avoid this for probable ytdl URLs because it can prevent the hook from triggering.
# Use memory:// M3U to preserve titles in MPV's playlist UI.
# Avoid this only for probable ytdl URLs because it can prevent the hook from triggering.
safe_title = title.replace("\n", " ").replace("\r", "") if title else None
if title and hydrus_target:
target_to_send = target
elif title and not _is_probable_ytdl_url(target):
if title and not _is_probable_ytdl_url(target):
# Sanitize title for M3U (remove newlines)
# Carry the store name for hash URLs so MPV.lyric can resolve the backend.
# This is especially important for local file-server URLs like /get_files/file?hash=...
@@ -2587,12 +2585,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if file_storage is None:
try:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
file_storage = Store(config)
file_storage = BackendRegistry(config)
except Exception as e:
debug(
f"Warning: Could not initialize Store registry: {e}",
f"Warning: Could not initialize backend registry: {e}",
file=sys.stderr
)
plugins/mpv/lyric.py
+12 -12
View File
@@ -1156,10 +1156,10 @@ def _infer_hydrus_store_from_url_target(*, target: str, config: dict) -> Optiona
return None
try:
from Store import Store as StoreRegistry
from PluginCore.backend_registry import BackendRegistry
reg = StoreRegistry(config, suppress_debug=True)
backends = [(name, reg[name]) for name in reg.list_backends()]
backend_registry = BackendRegistry(config, suppress_debug=True)
backends = [(name, backend_registry[name]) for name in backend_registry.list_backends()]
except Exception:
return None
@@ -1218,10 +1218,10 @@ def _resolve_store_backend_for_target(
return None, None
try:
from Store import Store as StoreRegistry
from PluginCore.backend_registry import BackendRegistry
reg = StoreRegistry(config, suppress_debug=True)
backend_names = list(reg.list_backends())
backend_registry = BackendRegistry(config, suppress_debug=True)
backend_names = list(backend_registry.list_backends())
except Exception:
return None, None
@@ -1229,7 +1229,7 @@ def _resolve_store_backend_for_target(
for name in backend_names:
try:
backend = reg[name]
backend = backend_registry[name]
except Exception:
continue
@@ -1363,16 +1363,16 @@ def run_auto_overlay(
# Import the Store registry once so each track change doesn't re-import the module.
try:
from Store import Store as _StoreRegistry # noqa: PLC0415
_store_cls: Any = _StoreRegistry
from PluginCore.backend_registry import BackendRegistry # noqa: PLC0415
_backend_registry_cls: Any = BackendRegistry
except Exception:
_store_cls = None
_backend_registry_cls = None
def _make_registry() -> Optional[Any]:
if _store_cls is None:
if _backend_registry_cls is None:
return None
try:
return _store_cls(cfg, suppress_debug=True)
return _backend_registry_cls(cfg, suppress_debug=True)
except Exception:
return None
plugins/mpv/mpv_ipc.py
+1 -1
View File
@@ -472,7 +472,7 @@ class MPV:
pipeline = f"file -download -url {_q(url)} -query {_q(f'format:{fmt}')}"
if store:
pipeline += f" | file -add -instance {_q(store)}"
pipeline += f" | file -add -plugin hydrusnetwork -instance {_q(store)}"
else:
pipeline += f" | file -add -plugin local -instance {_q(path or '')}"
plugins/mpv/pipeline_helper.py
+3 -3
View File
@@ -133,7 +133,7 @@ def _store_choices_payload(choices: Any) -> Optional[str]:
def _load_store_choices_from_config(*, force_reload: bool = False) -> list[str]:
from Store.registry import list_configured_backend_names # noqa: WPS433
from PluginCore.backend_registry import list_configured_backend_names # noqa: WPS433
cfg = reload_config() if force_reload else load_config()
return _normalize_store_choices(list_configured_backend_names(cfg or {}))
@@ -810,10 +810,10 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]:
"find-url",
"find_url"}:
try:
from Store import Store # noqa: WPS433
from PluginCore.backend_registry import BackendRegistry # noqa: WPS433
cfg = load_config() or {}
storage = Store(config=cfg, suppress_debug=True)
storage = BackendRegistry(config=cfg, suppress_debug=True)
raw_needles: list[str] = []
if isinstance(data, dict):
plugins/zeroxzero/__init__.py
+2 -2
View File
@@ -48,9 +48,9 @@ class ZeroXZero(Provider):
try:
pipe_obj = kwargs.get("pipe_obj")
if pipe_obj is not None:
from Store import Store
from PluginCore.backend_registry import BackendRegistry
Store(
BackendRegistry(
self.config,
suppress_debug=True
).try_add_url_for_pipe_object(pipe_obj,
tool/ytdlp.py
+9 -10
View File
@@ -2620,6 +2620,7 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, config: Optional[Dict[str, Any]] = None) -> Any:
from contextvars import copy_context
import threading
from typing import cast
@@ -2631,17 +2632,18 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co
except Exception as exc:
result_container[1] = exc
# Preserve the active pipeline ContextVar state so yt-dlp progress hooks in the
# worker thread can still see the shared Live UI and stage context.
download_context = copy_context()
# Use daemon=True so a hung download doesn't block process exit if the wall timeout hits.
thread = threading.Thread(target=_do_download, daemon=True)
thread = threading.Thread(target=lambda: download_context.run(_do_download), daemon=True)
thread.start()
start_time = time.monotonic()
# We use two timeouts:
# 1. Activity timeout (no progress updates for X seconds)
# 2. Hard wall-clock timeout (total time for this URL)
# The wall-clock timeout is slightly larger than the activity timeout
# to allow for slow-but-steady progress, up to a hard cap (e.g. 10 minutes).
wall_timeout = max(timeout_seconds * 2, 600)
# Keep only an activity timeout here. yt-dlp downloads can be legitimately slow
# for large media or constrained connections, and a fixed wall-clock cutoff can
# abort healthy downloads even while progress is still arriving.
_record_progress_activity(start_time)
try:
@@ -2659,9 +2661,6 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co
if now - last_activity > timeout_seconds:
raise DownloadError(f"Download activity timeout after {timeout_seconds} seconds for {opts.url}")
# Check hard wall-clock timeout
if now - start_time > wall_timeout:
raise DownloadError(f"Download hard timeout after {wall_timeout} seconds for {opts.url}")
finally:
_clear_progress_activity()