diff --git a/Store/_base.py b/PluginCore/backend_base.py similarity index 87% rename from Store/_base.py rename to PluginCore/backend_base.py index 08f7647..351dcad 100644 --- a/Store/_base.py +++ b/PluginCore/backend_base.py @@ -1,6 +1,7 @@ -"""Store backend base types. +"""Plugin-backed configured backend base types. -Concrete store implementations live in the `Store/` package. +Concrete storage backends live with their owning plugins and are resolved +through PluginCore.backend_registry. """ from __future__ import annotations @@ -10,12 +11,12 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -class Store(ABC): +class BackendBase(ABC): @classmethod def config_schema(cls) -> List[Dict[str, Any]]: - """Return configuration schema for this store. - + """Return configuration schema for this backend. + Returns a list of dicts: { "key": "PATH", @@ -32,27 +33,27 @@ class Store(ABC): @property def is_remote(self) -> bool: - """True if the store is a remote service (e.g. Hydrus) rather than local disk.""" + """True if this backend is a remote service rather than local disk.""" return False @property def prefer_defer_tags(self) -> bool: - """True if the store prefers tags to be applied after the file is added.""" + """True if this backend prefers tags to be applied after add_file().""" return False @property def supports_url_association(self) -> bool: - """True when this store supports associating URLs to files.""" + """True when this backend supports associating URLs to files.""" return False @property def supports_note_association(self) -> bool: - """True when this store supports per-file named notes.""" + """True when this backend supports per-file named notes.""" return False @property def supports_relationship_association(self) -> bool: - """True when this store supports file relationship links (king/alt/related).""" + """True when this backend supports file relationship links.""" return False @abstractmethod @@ -111,9 +112,8 @@ class Store(ABC): def delete_url_bulk( self, - items: List[Tuple[str, - List[str]]], - **kwargs: Any + items: List[Tuple[str, List[str]]], + **kwargs: Any, ) -> bool: """Optional bulk url deletion. @@ -162,7 +162,7 @@ class Store(ABC): file_identifier: str, name: str, text: str, - **kwargs: Any + **kwargs: Any, ) -> bool: """Add or replace a named note for a file.""" raise NotImplementedError @@ -173,7 +173,7 @@ class Store(ABC): *, ctx: Any, stage_is_last: bool = True, - **_kwargs: Any + **_kwargs: Any, ) -> bool: """Optional hook for handling `@N` selection semantics. @@ -187,4 +187,4 @@ class Store(ABC): @abstractmethod def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool: """Delete a named note for a file.""" - raise NotImplementedError + raise NotImplementedError \ No newline at end of file diff --git a/Store/registry.py b/PluginCore/backend_registry.py similarity index 50% rename from Store/registry.py rename to PluginCore/backend_registry.py index 22f1033..1267461 100644 --- a/Store/registry.py +++ b/PluginCore/backend_registry.py @@ -1,38 +1,31 @@ -"""Store registry. +"""Configured plugin-backed backend registry. -Concrete store implementations live in the `Store/` package. -This module is the single source of truth for store discovery. - -This registry is config-driven: -- Each store subtype (e.g. `hydrusnetwork`) maps to a concrete store class. -- Each store class advertises its required config keys via `StoreClass.__new__.keys`. -- Instances are created from config using those keys (case-insensitive lookup). +Backends are discovered from their owning plugins and instantiated from config. """ from __future__ import annotations import importlib import inspect -import pkgutil import re from typing import Any, Dict, Optional, Type from SYS.logger import debug from SYS.utils import expand_path -from Store._base import Store as BaseStore +from PluginCore.backend_base import BackendBase _SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$") -_DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None -_PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BaseStore]]] = {} +_PLUGIN_DISCOVERED_CLASSES_CACHE: Dict[str, Optional[Type[BackendBase]]] = {} # Backends that failed to initialize earlier in the current process. -# Keyed by (store_type, instance_key) where instance_key is the name used under config.store... +# Keyed by (backend_type, instance_key) where instance_key is the configured name +# under config.store... _FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {} -def _normalize_store_type(value: str) -> str: +def _normalize_backend_type(value: str) -> str: return "".join(ch.lower() for ch in str(value or "") if ch.isalnum()) @@ -44,59 +37,23 @@ def _get_case_insensitive(mapping: Dict[str, Any], key: str) -> Any: if key in mapping: return mapping[key] desired = _normalize_config_key(key) - for k, v in mapping.items(): - if _normalize_config_key(k) == desired: - return v + for current_key, value in mapping.items(): + if _normalize_config_key(current_key) == desired: + return value return None -def _discover_store_classes() -> Dict[str, Type[BaseStore]]: - """Discover store classes from the Store package. - - Convention: - - The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork). - """ - global _DISCOVERED_CLASSES_CACHE - if _DISCOVERED_CLASSES_CACHE is not None: - return _DISCOVERED_CLASSES_CACHE - - import Store as store_pkg - - discovered: Dict[str, Type[BaseStore]] = {} - for module_info in pkgutil.iter_modules(store_pkg.__path__): - module_name = module_info.name - if module_name.startswith(("_", "registry")): - continue - - try: - module = importlib.import_module(f"Store.{module_name}") - for _, obj in vars(module).items(): - if not inspect.isclass(obj): - continue - if obj is BaseStore: - continue - if not issubclass(obj, BaseStore): - continue - discovered[_normalize_store_type(obj.__name__)] = obj - except Exception as exc: - debug(f"[Store] Failed to import module '{module_name}': {exc}") - continue - - _DISCOVERED_CLASSES_CACHE = discovered - return discovered - - -def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]: - discovered: Dict[str, Type[BaseStore]] = {} +def _extract_backend_classes(owner: Any) -> Dict[str, Type[BackendBase]]: + discovered: Dict[str, Type[BackendBase]] = {} def _add_candidate(key: Any, candidate: Any) -> None: if not inspect.isclass(candidate): return - if candidate is BaseStore: + if candidate is BackendBase: return - if not issubclass(candidate, BaseStore): + if not issubclass(candidate, BackendBase): return - normalized = _normalize_store_type(str(key or candidate.__name__)) + normalized = _normalize_backend_type(str(key or candidate.__name__)) if normalized: discovered[normalized] = candidate @@ -125,8 +82,8 @@ def _extract_store_classes(owner: Any) -> Dict[str, Type[BaseStore]]: return discovered -def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]: - normalized = _normalize_store_type(store_type) +def _discover_plugin_backend_class(backend_type: str) -> Optional[Type[BackendBase]]: + normalized = _normalize_backend_type(backend_type) if not normalized: return None @@ -140,19 +97,19 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]: _PLUGIN_DISCOVERED_CLASSES_CACHE[normalized] = None return None - discovered: Dict[str, Type[BaseStore]] = {} + discovered: Dict[str, Type[BackendBase]] = {} backend_hook = getattr(plugin_module, "get_store_backend_classes", None) if callable(backend_hook): try: - discovered.update(_extract_store_classes(backend_hook())) + discovered.update(_extract_backend_classes(backend_hook())) except Exception as exc: - debug(f"[Store] Failed to load plugin store backends for '{normalized}': {exc}") + debug(f"[BackendRegistry] Failed to load plugin backends for '{normalized}': {exc}") - discovered.update(_extract_store_classes(getattr(plugin_module, "STORE_BACKENDS", None))) + discovered.update(_extract_backend_classes(getattr(plugin_module, "STORE_BACKENDS", None))) if normalized not in discovered: - discovered.update(_extract_store_classes(plugin_module)) + discovered.update(_extract_backend_classes(plugin_module)) resolved = discovered.get(normalized) if resolved is None and len(discovered) == 1: @@ -162,37 +119,24 @@ def _discover_plugin_store_class(store_type: str) -> Optional[Type[BaseStore]]: return resolved -def _resolve_store_class( - store_type: str, - classes_by_type: Optional[Dict[str, Type[BaseStore]]] = None, -) -> Optional[Type[BaseStore]]: - normalized = _normalize_store_type(store_type) +def _resolve_backend_class(backend_type: str) -> Optional[Type[BackendBase]]: + normalized = _normalize_backend_type(backend_type) if not normalized: return None - - plugin_resolved = _discover_plugin_store_class(normalized) - if plugin_resolved is not None: - return plugin_resolved - - discovered = classes_by_type if classes_by_type is not None else _discover_store_classes() - resolved = discovered.get(normalized) - if resolved is not None: - return resolved - return None + return _discover_plugin_backend_class(normalized) -def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]: - # Support new config_schema() schema - if hasattr(store_cls, "config_schema") and callable(store_cls.config_schema): +def _required_keys_for(backend_cls: Type[BackendBase]) -> list[str]: + if hasattr(backend_cls, "config_schema") and callable(backend_cls.config_schema): try: - schema = store_cls.config_schema() + schema = backend_cls.config_schema() keys = [] if isinstance(schema, list): for field in schema: if isinstance(field, dict) and field.get("required"): - k = field.get("key") - if k: - keys.append(str(k)) + key = field.get("key") + if key: + keys.append(str(key)) if keys: return keys except Exception: @@ -200,21 +144,14 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]: return [] -# Store type names that have been converted to providers-only. -# These should be silently skipped without warning. -_PROVIDER_ONLY_STORE_NAMES = frozenset(("debrid", "alldebrid")) +_PROVIDER_ONLY_BACKEND_NAMES = frozenset(("debrid", "alldebrid")) -def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]: - if isinstance(instance_config, dict): - cfg_dict = dict(instance_config) - else: - cfg_dict = {} +def _build_kwargs(backend_cls: Type[BackendBase], instance_name: str, instance_config: Any) -> Dict[str, Any]: + cfg_dict = dict(instance_config) if isinstance(instance_config, dict) else {} - required = _required_keys_for(store_cls) - - # If NAME is required but not present, allow the instance key to provide it. - if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None): + required = _required_keys_for(backend_cls) + if any(_normalize_config_key(key) == "NAME" for key in required) and _get_case_insensitive(cfg_dict, "NAME") is None: cfg_dict["NAME"] = str(instance_name) kwargs: Dict[str, Any] = {} @@ -228,28 +165,24 @@ def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_confi if missing: raise ValueError( - f"Missing required keys for {store_cls.__name__}: {', '.join(missing)}" + f"Missing required keys for {backend_cls.__name__}: {', '.join(missing)}" ) return kwargs -class Store: +class BackendRegistry: def __init__( self, - config: Optional[Dict[str, - Any]] = None, - suppress_debug: bool = False + config: Optional[Dict[str, Any]] = None, + suppress_debug: bool = False, ) -> None: self._config = config or {} self._suppress_debug = suppress_debug - self._backends: Dict[str, - BaseStore] = {} - self._backend_errors: Dict[str, - str] = {} - self._backend_types: Dict[str, - str] = {} + self._backends: Dict[str, BackendBase] = {} + self._backend_errors: Dict[str, str] = {} + self._backend_types: Dict[str, str] = {} self._load_backends() def _load_backends(self) -> None: @@ -258,61 +191,54 @@ class Store: store_cfg = {} self._backend_types = {} - classes_by_type = _discover_store_classes() - for raw_store_type, instances in store_cfg.items(): + for raw_backend_type, instances in store_cfg.items(): if not isinstance(instances, dict): continue - store_type = _normalize_store_type(str(raw_store_type)) - if store_type == "folder": + backend_type = _normalize_backend_type(str(raw_backend_type)) + if backend_type == "folder": continue - store_cls = _resolve_store_class(store_type, classes_by_type) - if store_cls is None: - # Skip provider-only names without debug warning - if store_type not in _PROVIDER_ONLY_STORE_NAMES and not self._suppress_debug: - debug(f"[Store] Unknown store type '{raw_store_type}'") + backend_cls = _resolve_backend_class(backend_type) + if backend_cls is None: + if backend_type not in _PROVIDER_ONLY_BACKEND_NAMES and not self._suppress_debug: + debug(f"[BackendRegistry] Unknown backend type '{raw_backend_type}'") continue for instance_name, instance_config in instances.items(): backend_name = str(instance_name) - - # If this backend already failed earlier in this process, skip re-instantiation. - cache_key = (store_type, str(instance_name)) + cache_key = (backend_type, str(instance_name)) cached_error = _FAILED_BACKEND_CACHE.get(cache_key) if cached_error: self._backend_errors[str(instance_name)] = str(cached_error) if isinstance(instance_config, dict): - override_name = _get_case_insensitive( - dict(instance_config), - "NAME" - ) + override_name = _get_case_insensitive(dict(instance_config), "NAME") if override_name: self._backend_errors[str(override_name)] = str(cached_error) continue + try: kwargs = _build_kwargs( - store_cls, + backend_cls, str(instance_name), - instance_config + instance_config, ) - # Convenience normalization for filesystem-like paths. for key in list(kwargs.keys()): if _normalize_config_key(key) in {"PATH", "LOCATION"}: kwargs[key] = str(expand_path(kwargs[key])) - backend = store_cls(**kwargs) + backend = backend_cls(**kwargs) backend_name = str(kwargs.get("NAME") or instance_name) self._backends[backend_name] = backend - self._backend_types[backend_name] = store_type + self._backend_types[backend_name] = backend_type except Exception as exc: err_text = str(exc) self._backend_errors[str(instance_name)] = err_text _FAILED_BACKEND_CACHE[cache_key] = err_text if not self._suppress_debug: debug( - f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}" + f"[BackendRegistry] Failed to register {backend_cls.__name__} instance '{instance_name}': {exc}" ) def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]: @@ -320,46 +246,41 @@ class Store: if requested in self._backends: return requested, None - requested_norm = _normalize_store_type(requested) + requested_norm = _normalize_backend_type(requested) ci_matches = [ name for name in self._backends - if _normalize_store_type(name) == requested_norm + if _normalize_backend_type(name) == requested_norm ] if len(ci_matches) == 1: return ci_matches[0], None if len(ci_matches) > 1: - return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}" + return None, f"Ambiguous backend alias '{backend_name}' matches {ci_matches}" type_matches = [ - name for name, store_type in self._backend_types.items() - if store_type == requested_norm + name for name, backend_type in self._backend_types.items() + if backend_type == requested_norm ] if len(type_matches) == 1: return type_matches[0], None if len(type_matches) > 1: return None, ( - f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}" + f"Ambiguous backend alias '{backend_name}' matches type '{requested_norm}': {type_matches}" ) prefix_matches = [ - name for name, store_type in self._backend_types.items() - if store_type.startswith(requested_norm) + name for name, backend_type in self._backend_types.items() + if backend_type.startswith(requested_norm) ] if len(prefix_matches) == 1: return prefix_matches[0], None if len(prefix_matches) > 1: return None, ( - f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}" + f"Ambiguous backend alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}" ) return None, None - # get_backend_instance implementation moved to the bottom of this file to avoid - # instantiating all backends during startup (see function `get_backend_instance`). - - # Duplicate _resolve_backend_name removed — the method is defined once earlier in the class. - def get_backend_error(self, backend_name: str) -> Optional[str]: return self._backend_errors.get(str(backend_name)) @@ -367,19 +288,17 @@ class Store: return sorted(self._backends.keys()) def list_searchable_backends(self) -> list[str]: - # De-duplicate backends by instance (aliases can point at the same object). def _rank(name: str) -> int: - n = str(name or "").strip().lower() - if n == "temp": + normalized_name = str(name or "").strip().lower() + if normalized_name == "temp": return 0 - if n == "default": + if normalized_name == "default": return 2 return 1 - chosen: Dict[int, - str] = {} + chosen: Dict[int, str] = {} for name, backend in self._backends.items(): - if type(backend).search is BaseStore.search: + if type(backend).search is BackendBase.search: continue key = id(backend) prev = chosen.get(key) @@ -387,27 +306,20 @@ class Store: chosen[key] = name return sorted(chosen.values()) - def __getitem__(self, backend_name: str) -> BaseStore: + def __getitem__(self, backend_name: str) -> BackendBase: resolved, err = self._resolve_backend_name(backend_name) if resolved: return self._backends[resolved] if err: - raise KeyError( - f"Unknown store backend: {backend_name}. {err}" - ) - raise KeyError( - f"Unknown store backend: {backend_name}. Available: {list(self._backends.keys())}" - ) + raise KeyError(f"Unknown backend: {backend_name}. {err}") + raise KeyError(f"Unknown backend: {backend_name}. Available: {list(self._backends.keys())}") def is_available(self, backend_name: str) -> bool: resolved, _err = self._resolve_backend_name(backend_name) return resolved is not None def try_add_url_for_pipe_object(self, pipe_obj: Any, url: str) -> bool: - """Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that store backend. - - Intended for providers to attach newly generated/hosted URLs back to an existing stored file. - """ + """Best-effort helper: if `pipe_obj` contains `store` + `hash`, add `url` to that backend.""" try: url_text = str(url or "").strip() if not url_text: @@ -442,41 +354,28 @@ class Store: def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]: - """Return backend instance names present in the provided config WITHOUT instantiating backends. - - This is a lightweight helper for CLI usage where we only need to know if a - configured backend exists (e.g., to distinguish a store name from a filesystem path) - without triggering backend initialization (which may perform network calls). - - Behaviour: - - Only includes store types that map to a discovered save backend class. - - Skips folder/provider-only/unknown entries so UI pickers do not surface - non-ingest destinations such as provider helpers. - - For each configured store type, returns the per-instance NAME override - (case-insensitive) when present, otherwise the instance key. - """ + """Return configured backend instance names without instantiating backends.""" try: - classes_by_type = _discover_store_classes() names: list[str] = [] for section_name in ("store", "plugin", "provider"): section_cfg = (config or {}).get(section_name) or {} if not isinstance(section_cfg, dict): continue - for raw_store_type, instances in section_cfg.items(): + for raw_backend_type, instances in section_cfg.items(): if not isinstance(instances, dict): continue - store_type = _normalize_store_type(str(raw_store_type)) - if store_type == "folder" or store_type in _PROVIDER_ONLY_STORE_NAMES: + backend_type = _normalize_backend_type(str(raw_backend_type)) + if backend_type == "folder" or backend_type in _PROVIDER_ONLY_BACKEND_NAMES: continue - store_cls = _resolve_store_class(store_type, classes_by_type) - if store_cls is None: + backend_cls = _resolve_backend_class(backend_type) + if backend_cls is None: continue for instance_name, instance_config in instances.items(): try: - _build_kwargs(store_cls, str(instance_name), instance_config) + _build_kwargs(backend_cls, str(instance_name), instance_config) except Exception: continue if isinstance(instance_config, dict): @@ -493,18 +392,15 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str] return [] -def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]: - """Instantiate and return a single store backend by configured name. - - This avoids creating all configured backends (and opening their DBs) - when only a single backend is needed (common in `get-file`/`get-metadata`). - The function first tries a lightweight match against raw config NAME/value to - avoid calling `_build_kwargs` (which can raise if keys are missing). - Returns None when no matching backend is found or instantiation fails. - """ +def get_backend_instance( + config: Optional[Dict[str, Any]], + backend_name: str, + *, + suppress_debug: bool = False, +) -> Optional[BackendBase]: + """Instantiate and return a single configured backend by name.""" if not backend_name: return None - classes_by_type = _discover_store_classes() desired = str(backend_name or "").strip().lower() for section_name in ("store", "plugin", "provider"): @@ -512,29 +408,28 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, if not isinstance(section_cfg, dict): continue - for raw_store_type, instances in section_cfg.items(): + for raw_backend_type, instances in section_cfg.items(): if not isinstance(instances, dict): continue - store_type = _normalize_store_type(str(raw_store_type)) - store_cls = _resolve_store_class(store_type, classes_by_type) - if store_cls is None: + backend_type = _normalize_backend_type(str(raw_backend_type)) + backend_cls = _resolve_backend_class(backend_type) + if backend_cls is None: continue - # Fast path: match using raw 'NAME' or 'name' in config without building full kwargs for instance_name, instance_cfg in instances.items(): candidate_alias = None if isinstance(instance_cfg, dict): - candidate_alias = ( - instance_cfg.get("NAME") or instance_cfg.get("name") - ) + candidate_alias = instance_cfg.get("NAME") or instance_cfg.get("name") candidate_alias = str(candidate_alias or instance_name).strip() if candidate_alias.lower() != desired: continue try: - kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) + kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg) except Exception as exc: if not suppress_debug: - debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}/{section_name}): {exc}") + debug( + f"[BackendRegistry] Can't build kwargs for '{instance_name}' ({backend_type}/{section_name}): {exc}" + ) return None try: for key in list(kwargs.keys()): @@ -543,17 +438,15 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, except Exception: pass try: - backend = store_cls(**kwargs) - return backend + return backend_cls(**kwargs) except Exception as exc: if not suppress_debug: - debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}") + debug(f"[BackendRegistry] Failed to instantiate backend '{candidate_alias}': {exc}") return None - # Fallback: build kwargs for each instance and compare resolved NAME for instance_name, instance_cfg in instances.items(): try: - kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) + kwargs = _build_kwargs(backend_cls, str(instance_name), instance_cfg) except Exception: continue alias = str(kwargs.get("NAME") or instance_name).strip() @@ -566,13 +459,20 @@ def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, except Exception: pass try: - backend = store_cls(**kwargs) - return backend + return backend_cls(**kwargs) except Exception as exc: if not suppress_debug: - debug(f"[Store] Failed to instantiate backend '{alias}': {exc}") + debug(f"[BackendRegistry] Failed to instantiate backend '{alias}': {exc}") return None if not suppress_debug: - debug(f"[Store] Backend '{backend_name}' not found in config") + debug(f"[BackendRegistry] Backend '{backend_name}' not found in config") return None + + +__all__ = [ + "BackendBase", + "BackendRegistry", + "get_backend_instance", + "list_configured_backend_names", +] \ No newline at end of file diff --git a/PluginCore/base.py b/PluginCore/base.py index 72aacad..d5f40d5 100644 --- a/PluginCore/base.py +++ b/PluginCore/base.py @@ -621,7 +621,7 @@ class Provider(ABC): raise NotImplementedError(f"Plugin '{self.name}' does not support upload") # ----------------------------------------------------------------------- - # Storage interface — mirrors Store._base.Store. + # Storage interface — mirrors PluginCore.backend_base.BackendBase. # Plugins that act as file repositories override these methods. # All raise NotImplementedError by default; override selectively. # ----------------------------------------------------------------------- diff --git a/SYS/cmdlet_spec.py b/SYS/cmdlet_spec.py index b136929..d25cb53 100644 --- a/SYS/cmdlet_spec.py +++ b/SYS/cmdlet_spec.py @@ -125,9 +125,9 @@ class SharedArgs: return try: - from Store.registry import Store as StoreRegistry + from PluginCore.backend_registry import BackendRegistry - registry = StoreRegistry(config=config, suppress_debug=True) + registry = BackendRegistry(config=config, suppress_debug=True) available = registry.list_backends() if available: SharedArgs._cached_available_stores = available diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 269dc94..893d9ac 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1730,21 +1730,21 @@ class PipelineExecutor: if store_keys: try: - from Store.registry import Store as StoreRegistry + from PluginCore.backend_registry import BackendRegistry - store_registry = StoreRegistry(config, suppress_debug=True) - _backend_names = list(store_registry.list_backends() or []) + backend_registry = BackendRegistry(config, suppress_debug=True) + _backend_names = list(backend_registry.list_backends() or []) _backend_by_lower = { str(n).lower(): str(n) for n in _backend_names if str(n).strip() } for name in store_keys: resolved_name = name - if not store_registry.is_available(resolved_name): + if not backend_registry.is_available(resolved_name): resolved_name = _backend_by_lower.get(str(name).lower(), name) - if not store_registry.is_available(resolved_name): + if not backend_registry.is_available(resolved_name): continue - backend = store_registry[resolved_name] + backend = backend_registry[resolved_name] selector = getattr(backend, "selector", None) if selector is None: continue diff --git a/Store/__init__.py b/Store/__init__.py deleted file mode 100644 index 77b76f9..0000000 --- a/Store/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from Store._base import Store as BaseStore -from Store.registry import Store - -__all__ = [ - "Store", - "BaseStore", -] diff --git a/TUI.py b/TUI.py index acd66f8..b2dfef9 100644 --- a/TUI.py +++ b/TUI.py @@ -53,7 +53,7 @@ from SYS.result_table import Table, extract_hash_value, extract_store_value, get from SYS.config import load_config # type: ignore # noqa: E402 from SYS.database import db -from Store.registry import Store as StoreRegistry # type: ignore # noqa: E402 +from PluginCore.backend_registry import BackendRegistry # type: ignore # noqa: E402 from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402 from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402 @@ -261,7 +261,7 @@ class TagEditorPopup(ModalScreen[None]): if not store_key or not hash_value: return None try: - registry = StoreRegistry(config=cfg, suppress_debug=True) + registry = BackendRegistry(config=cfg, suppress_debug=True) except Exception: return [] match = None @@ -984,9 +984,9 @@ class PipelineHubApp(App): stores: List[str] = [] try: - stores = StoreRegistry(config=cfg, suppress_debug=True).list_backends() + stores = BackendRegistry(config=cfg, suppress_debug=True).list_backends() except Exception: - logger.exception("Failed to list store backends from StoreRegistry") + logger.exception("Failed to list store backends from BackendRegistry") stores = [] # Always offer a reasonable default even if config is missing. @@ -1380,7 +1380,7 @@ class PipelineHubApp(App): cfg = {} try: - registry = StoreRegistry(config=cfg, suppress_debug=True) + registry = BackendRegistry(config=cfg, suppress_debug=True) except Exception: return None diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 9586bd5..300faec 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -1084,9 +1084,9 @@ def get_store_backend( registry = store_registry if registry is None: try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - registry = Store(config or {}, suppress_debug=suppress_debug) + registry = BackendRegistry(config or {}, suppress_debug=suppress_debug) except Exception as exc: return None, None, exc @@ -1110,7 +1110,7 @@ def get_preferred_store_backend( """Prefer a targeted backend instance before falling back to registry lookup.""" direct_exc: Optional[Exception] = None try: - from Store.registry import get_backend_instance + from PluginCore.backend_registry import get_backend_instance backend = get_backend_instance( config or {}, diff --git a/cmdlet/delete_url.py b/cmdlet/delete_url.py index 2de58aa..285de99 100644 --- a/cmdlet/delete_url.py +++ b/cmdlet/delete_url.py @@ -14,7 +14,7 @@ from ._shared import ( normalize_hash, ) from SYS.logger import log -from Store import Store +from PluginCore.backend_registry import BackendRegistry class Delete_Url(Cmdlet): @@ -106,7 +106,7 @@ class Delete_Url(Cmdlet): # Get backend and delete url try: - storage = Store(config) + storage = BackendRegistry(config) store_override = parsed.get("instance") diff --git a/cmdlet/file/add.py b/cmdlet/file/add.py index 108923f..f226aef 100644 --- a/cmdlet/file/add.py +++ b/cmdlet/file/add.py @@ -16,7 +16,7 @@ from SYS.pipeline_progress import PipelineProgress from SYS.result_publication import overlay_existing_result_table, publish_result_table from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS -from Store import Store +from PluginCore.backend_registry import BackendRegistry from API.HTTP import _download_direct_file from .. import _shared as sh @@ -45,21 +45,21 @@ SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS class _CommandDependencies: - """Command-scope cache for Store and plugin instances to avoid repeated instantiation.""" + """Command-scope cache for the backend registry and plugin instances.""" def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self._store: Optional[Store] = None + self._backend_registry: Optional[BackendRegistry] = None self._plugins: Dict[str, Any] = {} - def get_store(self) -> Optional[Store]: - """Lazily initialize and return the command-scope Store instance.""" - if self._store is None: + def get_backend_registry(self) -> Optional[BackendRegistry]: + """Lazily initialize and return the command-scope backend registry.""" + if self._backend_registry is None: try: - self._store = Store(self.config) + self._backend_registry = BackendRegistry(self.config) except Exception: - self._store = None - return self._store + self._backend_registry = None + return self._backend_registry def get_plugin(self, name: str) -> Optional[Any]: """Cached plugin lookup by name.""" @@ -236,7 +236,7 @@ class Add_File(Cmdlet): # Initialize command-scope dependency context (caches Store/plugins) deps = _CommandDependencies(config) - storage_registry = deps.get_store() + storage_registry = deps.get_backend_registry() path_arg = parsed.get("path") location = parsed.get("instance") @@ -354,8 +354,8 @@ class Add_File(Cmdlet): is_storage_backend_location = False if location: try: - store_for_lookup = storage_registry or deps.get_store() - is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None + backend_registry_for_lookup = storage_registry or deps.get_backend_registry() + is_storage_backend_location = Add_File._resolve_backend_by_name(backend_registry_for_lookup, str(location)) is not None except Exception: is_storage_backend_location = False @@ -697,8 +697,8 @@ class Add_File(Cmdlet): if location: try: - store = storage_registry or deps.get_store() - resolved_backend = Add_File._resolve_backend_by_name(store, str(location)) + backend_registry = storage_registry or deps.get_backend_registry() + resolved_backend = Add_File._resolve_backend_by_name(backend_registry, str(location)) if resolved_backend is not None: code = self._handle_storage_backend( item, @@ -845,7 +845,7 @@ class Add_File(Cmdlet): hash_values: List[str], config: Dict[str, Any], - store_instance: Optional[Store] = None, + store_instance: Optional[BackendRegistry] = None, ) -> Optional[List[Any]]: """Run search-file for a list of hashes and promote the table to a display overlay. @@ -1053,7 +1053,7 @@ class Add_File(Cmdlet): str]]], config: Dict[str, Any], - store_instance: Optional[Store] = None, + store_instance: Optional[BackendRegistry] = None, deps: Optional[_CommandDependencies] = None, ) -> None: """Persist relationships to backends that support relationships. @@ -1067,7 +1067,7 @@ class Add_File(Cmdlet): deps = _CommandDependencies(config) try: - store = store_instance if store_instance is not None else deps.get_store() + backend_registry = store_instance if store_instance is not None else deps.get_backend_registry() except Exception: return @@ -1076,7 +1076,7 @@ class Add_File(Cmdlet): continue try: - backend = store[str(backend_name)] + backend = backend_registry[str(backend_name)] except Exception: continue @@ -1356,9 +1356,9 @@ class Add_File(Cmdlet): try: if deps is None: deps = _CommandDependencies(config) - store = store_instance or deps.get_store() - - backend = Add_File._resolve_backend_by_name(store, r_store) + backend_registry = store_instance or deps.get_backend_registry() + + backend = Add_File._resolve_backend_by_name(backend_registry, r_store) if backend is not None: mp = backend.get_file(r_hash) if isinstance(mp, Path) and mp.exists(): @@ -1497,14 +1497,14 @@ class Add_File(Cmdlet): explicit_instance = str(instance_name or "").strip() or None try: - storage = store_instance if store_instance is not None else Store(config) + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) except Exception: - storage = None + backend_registry = None try: resolved_name, backend = resolver( explicit_instance, - storage=storage, + storage=backend_registry, require_explicit=bool(explicit_instance), ) except TypeError: @@ -1622,8 +1622,8 @@ class Add_File(Cmdlet): if deps is None: deps = _CommandDependencies(config) - store = store_instance or deps.get_store() - backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None + backend_registry = store_instance or deps.get_backend_registry() + backend = Add_File._resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None if backend is None: return None, None, None @@ -2475,10 +2475,23 @@ class Add_File(Cmdlet): List[str]]]]] = None, suppress_last_stage_overlay: bool = False, auto_search_file: bool = True, - store_instance: Optional[Store] = None, + store_instance: Optional[BackendRegistry] = None, ) -> int: """Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.).""" ##log(f"Adding file to storage backend '{backend_name}': {media_path.name}", file=sys.stderr) + pipeline_progress = PipelineProgress(ctx) + + def _set_status(text: str) -> None: + try: + pipeline_progress.set_status(f"{backend_name}: {text}") + except Exception: + pass + + def _clear_status() -> None: + try: + pipeline_progress.clear_status() + except Exception: + pass delete_after_effective = bool(delete_after) # ... (lines omitted for brevity but I need to keep them contextually correct) @@ -2510,11 +2523,11 @@ class Add_File(Cmdlet): pass try: - store = store_instance if store_instance is not None else Store(config) - backend, store, backend_exc = sh.get_preferred_store_backend( + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) + backend, backend_registry, backend_exc = sh.get_preferred_store_backend( config, backend_name, - store_registry=store, + store_registry=backend_registry, suppress_debug=True, ) if backend is None: @@ -2654,6 +2667,8 @@ class Add_File(Cmdlet): tag=upload_tags, url=[] if ((defer_url_association and url) or (not supports_url_association)) else url, file_hash=f_hash, + pipeline_progress=pipeline_progress, + transfer_label=title or media_path.name, ) ##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr) @@ -2704,6 +2719,7 @@ class Add_File(Cmdlet): try: adder = getattr(backend, "add_tag", None) if callable(adder): + _set_status("applying deferred tags") adder(resolved_hash, list(tags)) except Exception as exc: log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr) @@ -2724,88 +2740,43 @@ class Add_File(Cmdlet): try: # Folder.add_file already persists URLs, avoid extra DB traffic here. if not is_folder_backend: + _set_status("associating urls") backend.add_url(resolved_hash, list(url)) except Exception: pass # If a subtitle note was provided upstream (e.g., download-media writes notes.sub), # persist it automatically like add-note would. - sub_note = Add_File._get_note_text(result, pipe_obj, "sub") - if sub_note and supports_note_association: + def _write_note(note_name: str, note_text: Optional[str]) -> None: + if not note_text or not supports_note_association: + return try: setter = getattr(backend, "set_note", None) if callable(setter): - setter(resolved_hash, "sub", sub_note) + _set_status(f"writing {note_name} note") + setter(resolved_hash, note_name, note_text) except Exception as exc: debug_panel( "add-file note write failed", [ ("store", backend_name), ("hash", resolved_hash), - ("note", "sub"), + ("note", note_name), ("error", exc), ], border_style="yellow", ) - lyric_note = Add_File._get_note_text(result, pipe_obj, "lyric") - if lyric_note and supports_note_association: - try: - setter = getattr(backend, "set_note", None) - if callable(setter): - setter(resolved_hash, "lyric", lyric_note) - except Exception as exc: - debug_panel( - "add-file note write failed", - [ - ("store", backend_name), - ("hash", resolved_hash), - ("note", "lyric"), - ("error", exc), - ], - border_style="yellow", - ) - - chapters_note = Add_File._get_note_text(result, pipe_obj, "chapters") - if chapters_note and supports_note_association: - try: - setter = getattr(backend, "set_note", None) - if callable(setter): - setter(resolved_hash, "chapters", chapters_note) - except Exception as exc: - debug_panel( - "add-file note write failed", - [ - ("store", backend_name), - ("hash", resolved_hash), - ("note", "chapters"), - ("error", exc), - ], - border_style="yellow", - ) - - caption_note = Add_File._get_note_text(result, pipe_obj, "caption") - if caption_note and supports_note_association: - try: - setter = getattr(backend, "set_note", None) - if callable(setter): - setter(resolved_hash, "caption", caption_note) - except Exception as exc: - debug_panel( - "add-file note write failed", - [ - ("store", backend_name), - ("hash", resolved_hash), - ("note", "caption"), - ("error", exc), - ], - border_style="yellow", - ) + _write_note("sub", Add_File._get_note_text(result, pipe_obj, "sub")) + _write_note("lyric", Add_File._get_note_text(result, pipe_obj, "lyric")) + _write_note("chapters", Add_File._get_note_text(result, pipe_obj, "chapters")) + _write_note("caption", Add_File._get_note_text(result, pipe_obj, "caption")) meta: Dict[str, Any] = {} try: if not is_folder_backend: + _set_status("loading stored metadata") meta = backend.get_metadata(resolved_hash) or {} except Exception: meta = {} @@ -2886,9 +2857,11 @@ class Add_File(Cmdlet): media_path, delete_source=delete_after_effective ) + _clear_status() return 0 except Exception as exc: + _clear_status() log( f"❌ Failed to add file to backend '{backend_name}': {exc}", file=sys.stderr @@ -2907,12 +2880,12 @@ class Add_File(Cmdlet): List[str]]]], config: Dict[str, Any], - store_instance: Optional[Store] = None, + store_instance: Optional[BackendRegistry] = None, ) -> None: """Apply deferred URL associations in bulk, grouped per backend.""" try: - store = store_instance if store_instance is not None else Store(config) + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) except Exception: return @@ -2920,10 +2893,10 @@ class Add_File(Cmdlet): if not pairs: continue try: - backend, store, _exc = sh.get_store_backend( + backend, backend_registry, _exc = sh.get_store_backend( config, backend_name, - store_registry=store, + store_registry=backend_registry, ) if backend is None: continue @@ -2960,12 +2933,12 @@ class Add_File(Cmdlet): List[str]]]], config: Dict[str, Any], - store_instance: Optional[Store] = None, + store_instance: Optional[BackendRegistry] = None, ) -> None: """Apply deferred tag associations in bulk, grouped per backend.""" try: - store = store_instance if store_instance is not None else Store(config) + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) except Exception: return @@ -2974,7 +2947,7 @@ class Add_File(Cmdlet): pending or {}, bulk_method_name="add_tags_bulk", single_method_name="add_tag", - store_registry=store, + store_registry=backend_registry, pass_config_to_bulk=False, pass_config_to_single=False, ) diff --git a/cmdlet/file/delete.py b/cmdlet/file/delete.py index e045697..9008641 100644 --- a/cmdlet/file/delete.py +++ b/cmdlet/file/delete.py @@ -9,7 +9,7 @@ from pathlib import Path from SYS.logger import debug, log from PluginCore.registry import get_plugin -from Store import Store +from PluginCore.backend_registry import BackendRegistry from .. import _shared as sh from SYS import pipeline as ctx from SYS.result_table_helpers import add_row_columns @@ -156,7 +156,7 @@ class Delete_File(sh.Cmdlet): backend = None try: if instance: - registry = Store(config) + registry = BackendRegistry(config) if registry.is_available(str(store)): backend = registry[str(store)] except Exception: @@ -242,7 +242,7 @@ class Delete_File(sh.Cmdlet): try: # Re-use an already resolved backend when available. if backend is None: - registry = Store(config) + registry = BackendRegistry(config) if registry.is_available(str(store)): backend = registry[str(store)] diff --git a/cmdlet/file/download.py b/cmdlet/file/download.py index d790336..33e3e03 100644 --- a/cmdlet/file/download.py +++ b/cmdlet/file/download.py @@ -1661,7 +1661,7 @@ class Download_File(Cmdlet): """Initialize store registry and determine whether a Hydrus backend is usable.""" storage = None try: - from Store import Store as _Store + from PluginCore.backend_registry import BackendRegistry as _Store storage = _Store(config) except Exception: diff --git a/cmdlet/file/search.py b/cmdlet/file/search.py index 94860c8..4cbf72b 100644 --- a/cmdlet/file/search.py +++ b/cmdlet/file/search.py @@ -1865,8 +1865,8 @@ class search_file(Cmdlet): worker_id = str(uuid.uuid4()) - from Store import Store - storage_registry = Store(config=config or {}) + from PluginCore.backend_registry import BackendRegistry + storage_registry = BackendRegistry(config=config or {}) if not storage_registry.list_backends(): # Internal refreshes should not trigger config panels or stop progress. @@ -1911,8 +1911,8 @@ class search_file(Cmdlet): except Exception: pass - from Store.registry import list_configured_backend_names, get_backend_instance - from Store._base import Store as BaseStore + from PluginCore.backend_registry import list_configured_backend_names, get_backend_instance + from PluginCore.backend_base import BackendBase backend_to_search = storage_backend or None @@ -1929,13 +1929,13 @@ class search_file(Cmdlet): for h in hash_query: resolved_backend_name: Optional[str] = None resolved_backend = None - store_registry = None + backend_registry_cache = None for backend_name in backends_to_try: - backend, store_registry, _exc = get_preferred_store_backend( + backend, backend_registry_cache, _exc = get_preferred_store_backend( config, backend_name, - store_registry=store_registry, + store_registry=backend_registry_cache, suppress_debug=True, ) if backend is None: @@ -2128,7 +2128,7 @@ class search_file(Cmdlet): db.update_worker_status(worker_id, "error") return 1 - if type(target_backend).search is BaseStore.search: + if type(target_backend).search is BackendBase.search: log( f"Backend '{backend_to_search}' does not support searching", file=sys.stderr, @@ -2138,13 +2138,13 @@ class search_file(Cmdlet): results = target_backend.search(query, limit=limit) else: all_results = [] - store_registry = None + backend_registry_cache = None for backend_name in list_configured_backend_names(config or {}): try: - backend, store_registry, _exc = get_preferred_store_backend( + backend, backend_registry_cache, _exc = get_preferred_store_backend( config, backend_name, - store_registry=store_registry, + store_registry=backend_registry_cache, suppress_debug=True, ) if backend is None: @@ -2154,7 +2154,7 @@ class search_file(Cmdlet): searched_backends.append(backend_name) - if type(backend).search is BaseStore.search: + if type(backend).search is BackendBase.search: continue backend_results = backend.search( diff --git a/cmdlet/file/trim.py b/cmdlet/file/trim.py index 97d8990..1c0b8e6 100644 --- a/cmdlet/file/trim.py +++ b/cmdlet/file/trim.py @@ -14,6 +14,7 @@ from urllib.parse import urlparse from SYS.logger import log, debug from SYS.item_accessors import get_store_name from SYS.utils import sha256_file +from PluginCore.backend_registry import BackendRegistry from .. import _shared as sh Cmdlet = sh.Cmdlet @@ -166,8 +167,8 @@ def _persist_alt_relationship( ) -> None: """Persist directional alt -> king relationship in the given backend.""" try: - store = Store(config) - backend: Any = store[str(store_name)] + backend_registry = BackendRegistry(config) + backend: Any = backend_registry[str(store_name)] except Exception: return diff --git a/cmdlet/get_metadata.py b/cmdlet/get_metadata.py index ea4ec13..1e39216 100644 --- a/cmdlet/get_metadata.py +++ b/cmdlet/get_metadata.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, Sequence, Optional +from typing import Any, Dict, Sequence, Optional, List import json import sys @@ -235,6 +235,50 @@ class Get_Metadata(Cmdlet): columns_to_add.append(("Duration(s)", "")) add_row_columns(table, columns_to_add) + @staticmethod + def _extract_metadata_tags(metadata: Dict[str, Any]) -> List[str]: + tags: List[str] = [] + + def _append(tag_value: Any) -> None: + text = str(tag_value or "").strip() + if text and text not in tags: + tags.append(text) + + def _walk_tag_values(value: Any) -> None: + if isinstance(value, str): + _append(value) + return + if isinstance(value, dict): + for nested_value in value.values(): + _walk_tag_values(nested_value) + return + if isinstance(value, (list, tuple, set)): + for nested_value in value: + _walk_tag_values(nested_value) + + raw_tags = metadata.get("tags") + if isinstance(raw_tags, dict): + for service_data in raw_tags.values(): + if not isinstance(service_data, dict): + continue + matched_tag_key = False + for key, tag_mapping in service_data.items(): + if "tag" not in str(key).strip().lower(): + continue + matched_tag_key = True + _walk_tag_values(tag_mapping) + if not matched_tag_key: + _walk_tag_values(service_data) + elif isinstance(raw_tags, list): + for tag_value in raw_tags: + _append(tag_value) + + for key in ("tags_flat", "tag"): + raw_value = metadata.get(key) + _walk_tag_values(raw_value) + + return tags + def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: """Execute get-metadata cmdlet - retrieve and display file metadata. @@ -309,30 +353,29 @@ class Get_Metadata(Cmdlet): else: item_tags = [str(t) for t in item_tags] - # Extract tags from metadata response instead of making a separate get_tag() request - # This prevents duplicate API calls to Hydrus (metadata already includes tags) - metadata_tags = metadata.get("tags") - if isinstance(metadata_tags, dict): - # metadata["tags"] is {service_key: {service_data}} - for service_data in metadata_tags.values(): - if isinstance(service_data, dict): - display_tags = service_data.get("display_tags", {}) - if isinstance(display_tags, dict): - # display_tags is typically {status: tag_list} - for tag_list in display_tags.values(): - if isinstance(tag_list, list): - for t in tag_list: - ts = str(t) if t else "" - if ts and ts not in item_tags: - item_tags.append(ts) - # Check for title tag - if not get_field(result, "title") and ts.lower().startswith("title:"): - parts = ts.split(":", 1) - if len(parts) > 1: - title = parts[1].strip() - break # Only use first status level - if any(t for t in item_tags if str(t).lower().startswith("title:")): - break # Found title tag, stop searching services + metadata_tags = self._extract_metadata_tags(metadata) + if not metadata_tags: + get_tag = getattr(backend, "get_tag", None) + if callable(get_tag): + try: + backend_tags, _source = get_tag(file_hash, config=config) + metadata_tags = [ + str(tag) for tag in (backend_tags or []) + if str(tag or "").strip() + ] + except Exception: + metadata_tags = [] + + for tag_value in metadata_tags: + tag_text = str(tag_value or "").strip() + if not tag_text: + continue + if tag_text not in item_tags: + item_tags.append(tag_text) + if not get_field(result, "title") and tag_text.lower().startswith("title:"): + parts = tag_text.split(":", 1) + if len(parts) > 1: + title = parts[1].strip() # Extract metadata fields diff --git a/cmdlet/get_url.py b/cmdlet/get_url.py index 438a100..5db02aa 100644 --- a/cmdlet/get_url.py +++ b/cmdlet/get_url.py @@ -22,7 +22,7 @@ from SYS.logger import log from SYS.payload_builders import build_file_result_payload from SYS.result_publication import publish_result_table from SYS.result_table import Table -from Store import Store +from PluginCore.backend_registry import BackendRegistry from SYS import pipeline as ctx @@ -250,7 +250,7 @@ class Get_Url(Cmdlet): MAX_RESULTS = 256 try: - storage = Store(config) + storage = BackendRegistry(config) store_names = storage.list_backends() if hasattr(storage, "list_backends") else [] @@ -508,7 +508,7 @@ class Get_Url(Cmdlet): # Get backend and retrieve url try: - storage = Store(config) + storage = BackendRegistry(config) backend = storage[store_name] urls = backend.get_url(file_hash) diff --git a/cmdlet/metadata/tag_add.py b/cmdlet/metadata/tag_add.py index 7408226..ec2f166 100644 --- a/cmdlet/metadata/tag_add.py +++ b/cmdlet/metadata/tag_add.py @@ -13,7 +13,6 @@ from SYS.result_publication import publish_result_table from SYS import models from SYS import pipeline as ctx from .. import _shared as sh -from Store import Store # retained for test monkeypatch compatibility normalize_result_input = sh.normalize_result_input filter_results_by_temp = sh.filter_results_by_temp @@ -31,6 +30,7 @@ should_show_help = sh.should_show_help get_field = sh.get_field _FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$") +_DETAIL_PANEL_LIMIT = 9 def _normalize_title_for_extract(text: str) -> str: @@ -1191,7 +1191,13 @@ class Add_Tag(Cmdlet): subject = display_items[0] if len(display_items) == 1 else list(display_items) # Use helper to display items and make them @-selectable from ._shared import display_and_persist_items - display_and_persist_items(list(display_items), title="Result", subject=subject) + display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom" + display_and_persist_items( + list(display_items), + title="Result", + subject=subject, + display_type=display_type, + ) except Exception: pass diff --git a/cmdlet/metadata/tag_delete.py b/cmdlet/metadata/tag_delete.py index ac789ea..1cb9f41 100644 --- a/cmdlet/metadata/tag_delete.py +++ b/cmdlet/metadata/tag_delete.py @@ -22,6 +22,9 @@ get_field = sh.get_field from SYS.logger import debug, log +_DETAIL_PANEL_LIMIT = 9 + + def _matches_target( item: Any, target_hash: str | None, @@ -57,15 +60,56 @@ def _set_result_tags(result: Any, tags: list[str]) -> None: normalized = list(tags or []) set_field(result, "tag", normalized) + def _update_tag_columns(columns: Any) -> Any: + if not isinstance(columns, (list, tuple)): + return columns + + updated_columns = [] + changed = False + tag_value = ", ".join(normalized) + for column in columns: + if isinstance(column, tuple) and len(column) == 2: + label, existing_value = column + if str(label).strip().lower() in {"tag", "tags"}: + updated_columns.append((label, tag_value)) + changed = True + else: + updated_columns.append((label, existing_value)) + elif isinstance(column, list) and len(column) == 2: + label, existing_value = column + if str(label).strip().lower() in {"tag", "tags"}: + updated_columns.append([label, tag_value]) + changed = True + else: + updated_columns.append([label, existing_value]) + elif isinstance(column, dict): + label = column.get("name") or column.get("label") or column.get("key") + if str(label).strip().lower() in {"tag", "tags"}: + updated_column = dict(column) + updated_column["value"] = tag_value + updated_columns.append(updated_column) + changed = True + else: + updated_columns.append(column) + else: + updated_columns.append(column) + if not changed: + return columns + if isinstance(columns, tuple): + return tuple(updated_columns) + return updated_columns + if isinstance(result, dict): + result["tags_flat"] = list(normalized) if "tags" in result: result["tags"] = list(normalized) + result["columns"] = _update_tag_columns(result.get("columns")) for container_name in ("extra", "metadata", "full_metadata"): container = result.get(container_name) if not isinstance(container, dict): continue - if "tag" in container: - container["tag"] = list(normalized) + container["tag"] = list(normalized) + container["tags_flat"] = list(normalized) if "tags" in container: container["tags"] = list(normalized) return @@ -74,12 +118,23 @@ def _set_result_tags(result: Any, tags: list[str]) -> None: setattr(result, "tags", list(normalized)) except Exception: pass + try: + setattr(result, "tags_flat", list(normalized)) + except Exception: + pass + try: + columns = getattr(result, "columns", None) + updated_columns = _update_tag_columns(columns) + if updated_columns is not columns: + setattr(result, "columns", updated_columns) + except Exception: + pass for container_name in ("extra", "metadata", "full_metadata"): container = getattr(result, container_name, None) if not isinstance(container, dict): continue - if "tag" in container: - container["tag"] = list(normalized) + container["tag"] = list(normalized) + container["tags_flat"] = list(normalized) if "tags" in container: container["tags"] = list(normalized) @@ -514,6 +569,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # Process each item success_count = 0 + stage_ctx = ctx.get_stage_context() + is_last_stage = (stage_ctx is None) or bool( + getattr(stage_ctx, "is_last_stage", False) + ) + display_items: list[Any] = [] # If we have TagItems and no args, we are deleting the tags themselves # If we have Files (or other objects) and args, we are deleting tags FROM those files @@ -658,6 +718,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: if title_value: _apply_title_to_result(item, title_value) _refresh_result_table_tags(new_tags, h, store_str, path) + if is_last_stage: + display_items.append(item) try: ctx.emit(item) except Exception: @@ -667,12 +729,34 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: for item, item_hash, item_path, store_str in items_needing_individual: if _process_deletion(tags_arg, item_hash, item_path, store_str, config, result=item): success_count += 1 + if is_last_stage: + display_items.append(item) try: ctx.emit(item) except Exception: pass if success_count > 0: + if is_last_stage and display_items: + try: + subject = display_items[0] if len(display_items) == 1 else list(display_items) + from ._shared import display_and_persist_items + + display_type = "item" if len(display_items) <= _DETAIL_PANEL_LIMIT else "custom" + display_and_persist_items( + list(display_items), + title="Result", + subject=subject, + display_type=display_type, + ) + except Exception: + pass + + try: + if stage_ctx is not None: + stage_ctx.emits = [] + except Exception: + pass return 0 return 1 diff --git a/cmdlet/metadata/tag_get.py b/cmdlet/metadata/tag_get.py index 7016932..706656f 100644 --- a/cmdlet/metadata/tag_get.py +++ b/cmdlet/metadata/tag_get.py @@ -426,9 +426,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: backend = None if is_store_backed: try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - storage = Store(config, suppress_debug=True) + storage = BackendRegistry(config, suppress_debug=True) backend = storage[str(store_name)] except Exception: backend = None @@ -445,9 +445,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: store_for_scrape = get_field(result, "store", None) if file_hash_for_scrape and store_for_scrape: try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - storage = Store(config, suppress_debug=True) + storage = BackendRegistry(config, suppress_debug=True) backend = storage[str(store_for_scrape)] current_tags, _src = backend.get_tag(file_hash_for_scrape, config=config) if isinstance(current_tags, (list, tuple, set)) and current_tags: @@ -764,9 +764,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: ) return 0 try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - storage = Store(config, suppress_debug=True) + storage = BackendRegistry(config, suppress_debug=True) backend = storage[str(store_name)] ok = bool(backend.add_tag(file_hash, apply_tags, config=config)) if not ok: @@ -896,9 +896,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # Get tags using storage backend try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - storage = Store(config, suppress_debug=True) + storage = BackendRegistry(config, suppress_debug=True) backend = storage[store_name] current, source = backend.get_tag(file_hash, config=config) current = list(current or []) diff --git a/cmdlet/metadata/url_add.py b/cmdlet/metadata/url_add.py index 5986766..fd86262 100644 --- a/cmdlet/metadata/url_add.py +++ b/cmdlet/metadata/url_add.py @@ -6,7 +6,7 @@ import sys from SYS import pipeline as ctx from .. import _shared as sh from SYS.logger import log -from Store import Store +from PluginCore.backend_registry import BackendRegistry class Add_Url(sh.Cmdlet): @@ -117,7 +117,7 @@ class Add_Url(sh.Cmdlet): # Get backend and add url try: - storage = Store(config) + storage = BackendRegistry(config) # Build batches per store. store_override = parsed.get("instance") diff --git a/plugins/README.md b/plugins/README.md index 8d0a0a8..baf77fc 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -65,4 +65,4 @@ Bundled walkthrough: - The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance `, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance ` uploads. - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). - The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance `, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance ` uploads. -- The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins//api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims. \ No newline at end of file +- The repo also includes a built-in HydrusNetwork plugin in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its configured-backend adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins//api/` package shape is the intended pattern for plugin-owned API helpers going forward. The plugin resolves configured Hydrus instances directly from plugin config instead of routing back through `PluginCore.backend_registry`; the proxy exists only so generic backend callers can still target configured Hydrus instances. \ No newline at end of file diff --git a/plugins/fileio/__init__.py b/plugins/fileio/__init__.py index 817ab3f..deeacd6 100644 --- a/plugins/fileio/__init__.py +++ b/plugins/fileio/__init__.py @@ -214,9 +214,9 @@ class FileIO(Provider): try: pipe_obj = kwargs.get("pipe_obj") if pipe_obj is not None: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - Store( + BackendRegistry( self.config, suppress_debug=True ).try_add_url_for_pipe_object(pipe_obj, diff --git a/plugins/internetarchive/__init__.py b/plugins/internetarchive/__init__.py index ed68fc4..9b39615 100644 --- a/plugins/internetarchive/__init__.py +++ b/plugins/internetarchive/__init__.py @@ -1296,9 +1296,9 @@ class InternetArchive(Provider): try: if pipe_obj is not None: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - Store( + BackendRegistry( self.config, suppress_debug=True ).try_add_url_for_pipe_object(pipe_obj, diff --git a/plugins/matrix/__init__.py b/plugins/matrix/__init__.py index b381d08..83559ce 100644 --- a/plugins/matrix/__init__.py +++ b/plugins/matrix/__init__.py @@ -636,10 +636,10 @@ class Matrix(TablePluginMixin, Provider): try: pipe_obj = kwargs.get("pipe_obj") if pipe_obj is not None: - from Store import Store + from PluginCore.backend_registry import BackendRegistry # Prefer the direct media download URL for storage backends. - Store( + BackendRegistry( self.config, suppress_debug=True ).try_add_url_for_pipe_object( diff --git a/plugins/mpv/LUA/main.lua b/plugins/mpv/LUA/main.lua index 98280d2..b91d00a 100644 --- a/plugins/mpv/LUA/main.lua +++ b/plugins/mpv/LUA/main.lua @@ -901,7 +901,7 @@ function M._load_store_choices_direct_async(cb) 'from SYS.logger import set_thread_stream', 'set_thread_stream(sys.stderr)', 'from SYS.config import load_config', - 'from Store.registry import list_configured_backend_names', + 'from PluginCore.backend_registry import list_configured_backend_names', 'config = load_config()', 'choices = list_configured_backend_names(config) or []', 'sys.stdout.write(json.dumps({"choices": choices}, ensure_ascii=False))', @@ -2842,7 +2842,7 @@ local function _start_screenshot_store_save(store, out_path, tags) if screenshot_url == '' or not screenshot_url:match('^https?://') then screenshot_url = '' end - local cmd = 'file -add -store ' .. quote_pipeline_arg(store) + local cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store) .. ' -path ' .. quote_pipeline_arg(out_path) if screenshot_url ~= '' then cmd = cmd .. ' -url ' .. quote_pipeline_arg(screenshot_url) @@ -5864,7 +5864,7 @@ mp.register_script_message('medios-download-pick-store', function(json) local pipeline_cmd = 'file -download -url ' .. quote_pipeline_arg(url) .. ' -query ' .. quote_pipeline_arg(query) - .. ' | file -add -store ' .. quote_pipeline_arg(store) + .. ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store) _set_selected_store(store) _queue_pipeline_in_repl( @@ -6331,16 +6331,16 @@ local function _start_trim_with_range(range) pipeline_cmd = 'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. - ' | file -add -path ' .. quote_pipeline_arg(output_path) .. - ' -store "' .. selected_store .. '"' .. + ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) .. + ' -path ' .. quote_pipeline_arg(output_path) .. ' | add-relationship -store "' .. selected_store .. '"' .. ' -to-hash ' .. quote_pipeline_arg(store_hash.hash) else pipeline_cmd = 'tag -get -emit -store ' .. quote_pipeline_arg(store_hash.store) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. - ' | file -add -path ' .. quote_pipeline_arg(output_path) .. - ' -store "' .. store_hash.store .. '"' .. + ' | file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(store_hash.store) .. + ' -path ' .. quote_pipeline_arg(output_path) .. ' | add-relationship -store "' .. store_hash.store .. '"' .. ' -to-hash ' .. quote_pipeline_arg(store_hash.hash) end @@ -6350,8 +6350,8 @@ local function _start_trim_with_range(range) if selected_store then _lua_log('trim: building file -add command to selected_store=' .. selected_store) -- Don't add title if empty - the file path will be used as title by default - pipeline_cmd = 'file -add -path ' .. quote_pipeline_arg(output_path) .. - ' -store "' .. selected_store .. '"' + pipeline_cmd = 'file -add -plugin hydrusnetwork -instance ' .. quote_pipeline_arg(selected_store) .. + ' -path ' .. quote_pipeline_arg(output_path) _lua_log('trim: pipeline_cmd=' .. pipeline_cmd) else mp.osd_message('Trim complete: ' .. output_path, 5) diff --git a/plugins/mpv/commands.py b/plugins/mpv/commands.py index cd4fddb..30c12ad 100644 --- a/plugins/mpv/commands.py +++ b/plugins/mpv/commands.py @@ -816,7 +816,7 @@ def _prefetch_notes_async( set_notes_prefetch_pending, store_cached_notes, ) - from Store import Store + from PluginCore.backend_registry import BackendRegistry cached = load_cached_notes(store, file_hash, config=cfg) if cached is not None: @@ -824,7 +824,7 @@ def _prefetch_notes_async( set_notes_prefetch_pending(store, file_hash, True) - registry = Store(cfg, suppress_debug=True) + registry = BackendRegistry(cfg, suppress_debug=True) if not registry.is_available(str(store)): return backend = registry[str(store)] @@ -1674,14 +1674,14 @@ def _queue_items( except Exception: hydrus_url = None - # Initialize Store registry for path resolution + # Initialize backend registry for path resolution file_storage = None try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - file_storage = Store(config or {}) + file_storage = BackendRegistry(config or {}) except Exception as e: - debug(f"Warning: Could not initialize Store registry: {e}", file=sys.stderr) + debug(f"Warning: Could not initialize backend registry: {e}", file=sys.stderr) _schedule_notes_prefetch(items, config) @@ -1831,12 +1831,10 @@ def _queue_items( continue new_targets.add(norm_key) - # Use memory:// M3U hack to pass title to MPV. - # Avoid this for probable ytdl URLs because it can prevent the hook from triggering. + # Use memory:// M3U to preserve titles in MPV's playlist UI. + # Avoid this only for probable ytdl URLs because it can prevent the hook from triggering. safe_title = title.replace("\n", " ").replace("\r", "") if title else None - if title and hydrus_target: - target_to_send = target - elif title and not _is_probable_ytdl_url(target): + if title and not _is_probable_ytdl_url(target): # Sanitize title for M3U (remove newlines) # Carry the store name for hash URLs so MPV.lyric can resolve the backend. # This is especially important for local file-server URLs like /get_files/file?hash=... @@ -2587,12 +2585,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: if file_storage is None: try: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - file_storage = Store(config) + file_storage = BackendRegistry(config) except Exception as e: debug( - f"Warning: Could not initialize Store registry: {e}", + f"Warning: Could not initialize backend registry: {e}", file=sys.stderr ) diff --git a/plugins/mpv/lyric.py b/plugins/mpv/lyric.py index 9b96f41..da9a236 100644 --- a/plugins/mpv/lyric.py +++ b/plugins/mpv/lyric.py @@ -1156,10 +1156,10 @@ def _infer_hydrus_store_from_url_target(*, target: str, config: dict) -> Optiona return None try: - from Store import Store as StoreRegistry + from PluginCore.backend_registry import BackendRegistry - reg = StoreRegistry(config, suppress_debug=True) - backends = [(name, reg[name]) for name in reg.list_backends()] + backend_registry = BackendRegistry(config, suppress_debug=True) + backends = [(name, backend_registry[name]) for name in backend_registry.list_backends()] except Exception: return None @@ -1218,10 +1218,10 @@ def _resolve_store_backend_for_target( return None, None try: - from Store import Store as StoreRegistry + from PluginCore.backend_registry import BackendRegistry - reg = StoreRegistry(config, suppress_debug=True) - backend_names = list(reg.list_backends()) + backend_registry = BackendRegistry(config, suppress_debug=True) + backend_names = list(backend_registry.list_backends()) except Exception: return None, None @@ -1229,7 +1229,7 @@ def _resolve_store_backend_for_target( for name in backend_names: try: - backend = reg[name] + backend = backend_registry[name] except Exception: continue @@ -1363,16 +1363,16 @@ def run_auto_overlay( # Import the Store registry once so each track change doesn't re-import the module. try: - from Store import Store as _StoreRegistry # noqa: PLC0415 - _store_cls: Any = _StoreRegistry + from PluginCore.backend_registry import BackendRegistry # noqa: PLC0415 + _backend_registry_cls: Any = BackendRegistry except Exception: - _store_cls = None + _backend_registry_cls = None def _make_registry() -> Optional[Any]: - if _store_cls is None: + if _backend_registry_cls is None: return None try: - return _store_cls(cfg, suppress_debug=True) + return _backend_registry_cls(cfg, suppress_debug=True) except Exception: return None diff --git a/plugins/mpv/mpv_ipc.py b/plugins/mpv/mpv_ipc.py index c6392fa..10a9bf4 100644 --- a/plugins/mpv/mpv_ipc.py +++ b/plugins/mpv/mpv_ipc.py @@ -472,7 +472,7 @@ class MPV: pipeline = f"file -download -url {_q(url)} -query {_q(f'format:{fmt}')}" if store: - pipeline += f" | file -add -instance {_q(store)}" + pipeline += f" | file -add -plugin hydrusnetwork -instance {_q(store)}" else: pipeline += f" | file -add -plugin local -instance {_q(path or '')}" diff --git a/plugins/mpv/pipeline_helper.py b/plugins/mpv/pipeline_helper.py index a3cb45b..7619aac 100644 --- a/plugins/mpv/pipeline_helper.py +++ b/plugins/mpv/pipeline_helper.py @@ -133,7 +133,7 @@ def _store_choices_payload(choices: Any) -> Optional[str]: def _load_store_choices_from_config(*, force_reload: bool = False) -> list[str]: - from Store.registry import list_configured_backend_names # noqa: WPS433 + from PluginCore.backend_registry import list_configured_backend_names # noqa: WPS433 cfg = reload_config() if force_reload else load_config() return _normalize_store_choices(list_configured_backend_names(cfg or {})) @@ -810,10 +810,10 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]: "find-url", "find_url"}: try: - from Store import Store # noqa: WPS433 + from PluginCore.backend_registry import BackendRegistry # noqa: WPS433 cfg = load_config() or {} - storage = Store(config=cfg, suppress_debug=True) + storage = BackendRegistry(config=cfg, suppress_debug=True) raw_needles: list[str] = [] if isinstance(data, dict): diff --git a/plugins/zeroxzero/__init__.py b/plugins/zeroxzero/__init__.py index 6fc09eb..390f813 100644 --- a/plugins/zeroxzero/__init__.py +++ b/plugins/zeroxzero/__init__.py @@ -48,9 +48,9 @@ class ZeroXZero(Provider): try: pipe_obj = kwargs.get("pipe_obj") if pipe_obj is not None: - from Store import Store + from PluginCore.backend_registry import BackendRegistry - Store( + BackendRegistry( self.config, suppress_debug=True ).try_add_url_for_pipe_object(pipe_obj, diff --git a/tool/ytdlp.py b/tool/ytdlp.py index 958a1c6..d685ca4 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -2620,6 +2620,7 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, config: Optional[Dict[str, Any]] = None) -> Any: + from contextvars import copy_context import threading from typing import cast @@ -2631,17 +2632,18 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co except Exception as exc: result_container[1] = exc + # Preserve the active pipeline ContextVar state so yt-dlp progress hooks in the + # worker thread can still see the shared Live UI and stage context. + download_context = copy_context() + # Use daemon=True so a hung download doesn't block process exit if the wall timeout hits. - thread = threading.Thread(target=_do_download, daemon=True) + thread = threading.Thread(target=lambda: download_context.run(_do_download), daemon=True) thread.start() start_time = time.monotonic() - # We use two timeouts: - # 1. Activity timeout (no progress updates for X seconds) - # 2. Hard wall-clock timeout (total time for this URL) - # The wall-clock timeout is slightly larger than the activity timeout - # to allow for slow-but-steady progress, up to a hard cap (e.g. 10 minutes). - wall_timeout = max(timeout_seconds * 2, 600) + # Keep only an activity timeout here. yt-dlp downloads can be legitimately slow + # for large media or constrained connections, and a fixed wall-clock cutoff can + # abort healthy downloads even while progress is still arriving. _record_progress_activity(start_time) try: @@ -2659,9 +2661,6 @@ def _download_with_timeout(opts: DownloadOptions, timeout_seconds: int = 300, co if now - last_activity > timeout_seconds: raise DownloadError(f"Download activity timeout after {timeout_seconds} seconds for {opts.url}") - # Check hard wall-clock timeout - if now - start_time > wall_timeout: - raise DownloadError(f"Download hard timeout after {wall_timeout} seconds for {opts.url}") finally: _clear_progress_activity()