continuing refactor

This commit is contained in:
2026-05-03 21:20:05 -07:00
parent 77cab1bd27
commit 5534812426
50 changed files with 1004 additions and 428 deletions
+6 -6
View File
@@ -506,17 +506,17 @@ class CmdletIntrospection:
if normalized_arg == "plugin": if normalized_arg == "plugin":
canonical_cmd = (cmd_name or "").replace("_", "-").lower() canonical_cmd = (cmd_name or "").replace("_", "-").lower()
try: try:
from ProviderCore.registry import list_plugin_names_with_capability from ProviderCore.registry import list_configured_plugin_names_with_capability
except Exception: except Exception:
list_plugin_names_with_capability = None # type: ignore list_configured_plugin_names_with_capability = None # type: ignore
plugin_choices: List[str] = [] plugin_choices: List[str] = []
if canonical_cmd in {"add-file"} and list_plugin_names_with_capability is not None: if canonical_cmd in {"add-file"} and list_configured_plugin_names_with_capability is not None:
return list_plugin_names_with_capability("upload") or [] return list_configured_plugin_names_with_capability("upload", config) or []
if list_plugin_names_with_capability is not None: if list_configured_plugin_names_with_capability is not None:
plugin_choices = list_plugin_names_with_capability("search") or [] plugin_choices = list_configured_plugin_names_with_capability("search", config) or []
if plugin_choices: if plugin_choices:
return plugin_choices return plugin_choices
+90 -5
View File
@@ -161,6 +161,17 @@ class Provider(ABC):
# generic "file host" plugins via `add-file -plugin ...`. # generic "file host" plugins via `add-file -plugin ...`.
EXPOSE_AS_FILE_PROVIDER: bool = True EXPOSE_AS_FILE_PROVIDER: bool = True
# Set to True for plugins that support multiple named instances in config.
# When True, config is expected at config["plugin"][<PLUGIN_NAME>][<instance_name>]
# rather than config["plugin"][<PLUGIN_NAME>] directly.
# Examples: hydrusnetwork (home/work), matrix (personal/work), ftp.
MULTI_INSTANCE: bool = False
# Declare which top-level cmdlet names this plugin handles.
# Cmdlet dispatch and capability discovery use this to route operations.
# Example: frozenset({"add-file", "get-file", "get-tag", "search-file"})
SUPPORTED_CMDLETS: frozenset = frozenset()
def __init__(self, config: Optional[Dict[str, Any]] = None): def __init__(self, config: Optional[Dict[str, Any]] = None):
self.config = config or {} self.config = config or {}
self.name = str( self.name = str(
@@ -312,11 +323,21 @@ class Provider(ABC):
def plugin_config_root(self) -> Dict[str, Any]: def plugin_config_root(self) -> Dict[str, Any]:
if not isinstance(self.config, dict): if not isinstance(self.config, dict):
return {} return {}
provider_cfg = self.config.get("provider") # Check plugin/provider section first (preferred new format)
if not isinstance(provider_cfg, dict): for section in ("plugin", "provider"):
return {} section_cfg = self.config.get(section)
entry = provider_cfg.get(self.plugin_config_key()) if isinstance(section_cfg, dict):
return dict(entry) if isinstance(entry, dict) else {} entry = section_cfg.get(self.plugin_config_key())
if isinstance(entry, dict):
return dict(entry)
# Backward compat: fall back to store section.
# store config uses {type: {instance: {key: val}}} — one level deeper.
store_cfg = self.config.get("store")
if isinstance(store_cfg, dict):
store_entries = store_cfg.get(self.plugin_config_key())
if isinstance(store_entries, dict):
return dict(store_entries)
return {}
def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]: def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]:
entry = self.plugin_config_root() entry = self.plugin_config_root()
@@ -599,6 +620,70 @@ class Provider(ABC):
"""Upload a file and return a URL or identifier.""" """Upload a file and return a URL or identifier."""
raise NotImplementedError(f"Plugin '{self.name}' does not support upload") raise NotImplementedError(f"Plugin '{self.name}' does not support upload")
# -----------------------------------------------------------------------
# Storage interface — mirrors Store._base.Store.
# Plugins that act as file repositories override these methods.
# All raise NotImplementedError by default; override selectively.
# -----------------------------------------------------------------------
@property
def is_remote(self) -> bool:
"""True if this plugin stores files on a remote service."""
return False
@property
def prefer_defer_tags(self) -> bool:
"""True if tag writes should be deferred until after file ingest."""
return False
def add_file(self, file_path: Path, **kwargs: Any) -> str:
"""Ingest a file and return its canonical hash."""
raise NotImplementedError(f"Plugin '{self.name}' does not support add_file")
def get_file(self, file_hash: str, **kwargs: Any) -> Optional[Path]:
"""Retrieve a stored file by hash, returning a local Path or None."""
raise NotImplementedError(f"Plugin '{self.name}' does not support get_file")
def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
"""Return metadata dict for a stored file."""
raise NotImplementedError(f"Plugin '{self.name}' does not support get_metadata")
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
"""Return (tags, hash) for a stored file identifier."""
raise NotImplementedError(f"Plugin '{self.name}' does not support get_tag")
def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
"""Add tags to a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support add_tag")
def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool:
"""Remove tags from a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support delete_tag")
def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]:
"""Return associated URLs for a stored file."""
raise NotImplementedError(f"Plugin '{self.name}' does not support get_url")
def add_url(self, file_identifier: str, urls: List[str], **kwargs: Any) -> bool:
"""Associate URLs with a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support add_url")
def delete_url(self, file_identifier: str, urls: List[str], **kwargs: Any) -> bool:
"""Remove URL associations from a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support delete_url")
def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]:
"""Return notes dict (name -> text) for a stored file."""
raise NotImplementedError(f"Plugin '{self.name}' does not support get_note")
def set_note(self, file_identifier: str, name: str, text: str, **kwargs: Any) -> bool:
"""Write a named note on a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support set_note")
def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool:
"""Delete a named note from a stored file. Returns True on success."""
raise NotImplementedError(f"Plugin '{self.name}' does not support delete_note")
def validate(self) -> bool: def validate(self) -> bool:
"""Check if the plugin is available and properly configured.""" """Check if the plugin is available and properly configured."""
+84
View File
@@ -150,6 +150,20 @@ class PluginInfo:
exposed = True exposed = True
return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload) return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload)
@property
def is_multi_instance(self) -> bool:
"""True if the plugin declares MULTI_INSTANCE = True."""
return bool(getattr(self.plugin_class, "MULTI_INSTANCE", False))
@property
def supported_cmdlets(self) -> frozenset:
"""Frozenset of cmdlet names this plugin declares support for."""
raw = getattr(self.plugin_class, "SUPPORTED_CMDLETS", frozenset())
try:
return frozenset(str(c) for c in raw)
except Exception:
return frozenset()
class PluginRegistry: class PluginRegistry:
"""Handles discovery, registration, and lookup of built-in and external plugins.""" """Handles discovery, registration, and lookup of built-in and external plugins."""
@@ -433,6 +447,42 @@ class PluginRegistry:
def has_name(self, name: str) -> bool: def has_name(self, name: str) -> bool:
return self.get(name) is not None return self.get(name) is not None
def get_plugins_for_cmdlet(self, cmdlet_name: str) -> List[PluginInfo]:
"""Return all plugins that declare support for the given cmdlet name."""
self.discover()
target = str(cmdlet_name or "").strip().lower()
return [
info for info in self._infos.values()
if target in info.supported_cmdlets
]
def list_storage_plugin_instances(
self,
config: Optional[Dict[str, Any]] = None,
) -> Dict[str, List[str]]:
"""Return {plugin_name: [instance_name, ...]} for all MULTI_INSTANCE storage plugins.
Instance names come from the plugin's resolved config (plugin/provider/store sections).
Plugins with no configured instances are omitted.
"""
self.discover()
result: Dict[str, List[str]] = {}
for info in self._infos.values():
if not info.is_multi_instance:
continue
if not info.supported_cmdlets.intersection(
{"add-file", "get-file", "get-tag", "add-tag"}
):
continue
try:
instance = info.plugin_class(config or {})
instances = instance.configured_instances()
if instances:
result[info.canonical_name] = instances
except Exception:
pass
return result
def _sync_subclasses(self) -> None: def _sync_subclasses(self) -> None:
"""Walk all plugin subclasses in memory and register them.""" """Walk all plugin subclasses in memory and register them."""
def _walk(cls: Type[Provider]) -> None: def _walk(cls: Type[Provider]) -> None:
@@ -691,6 +741,39 @@ def list_plugin_names_with_capability(capability: str) -> List[str]:
) )
def list_configured_plugin_names_with_capability(
capability: str,
config: Optional[Dict[str, Any]] = None,
) -> List[str]:
"""Return plugin names that support `capability` AND have configuration present.
For MULTI_INSTANCE plugins (e.g. hydrusnetwork, ftp) the plugin must have at
least one configured instance. For single-instance plugins the plugin's section
must exist under config["plugin"] or config["provider"].
"""
cfg = config or {}
plugin_section: Dict[str, Any] = cfg.get("plugin") or {} # type: ignore[assignment]
provider_section: Dict[str, Any] = cfg.get("provider") or {} # type: ignore[assignment]
result: List[str] = []
for info in REGISTRY.iter_plugins():
if not _info_supports_capability(info, capability):
continue
name = info.canonical_name
if info.is_multi_instance:
try:
instances = info.plugin_class(cfg).configured_instances()
if instances:
result.append(name)
except Exception:
pass
else:
pname = name.lower()
if isinstance(plugin_section.get(pname), dict) or isinstance(provider_section.get(pname), dict):
result.append(name)
return sorted(result)
def match_plugin_name_for_url(url: str) -> Optional[str]: def match_plugin_name_for_url(url: str) -> Optional[str]:
raw_url = str(url or "").strip() raw_url = str(url or "").strip()
raw_url_lower = raw_url.lower() raw_url_lower = raw_url.lower()
@@ -907,6 +990,7 @@ __all__ = [
"get_plugin_with_capability", "get_plugin_with_capability",
"list_plugins_with_capability", "list_plugins_with_capability",
"list_plugin_names_with_capability", "list_plugin_names_with_capability",
"list_configured_plugin_names_with_capability",
"match_plugin_name_for_url", "match_plugin_name_for_url",
"get_plugin_for_url", "get_plugin_for_url",
"list_selection_url_prefixes", "list_selection_url_prefixes",
+1 -1
View File
@@ -138,7 +138,7 @@ def _validate_add_note_requires_add_file_order(raw: str) -> Optional[SyntaxError
# If add-note occurs before any add-file stage, it must be explicitly targeted. # If add-note occurs before any add-file stage, it must be explicitly targeted.
if any(pos > i for pos in add_file_positions): if any(pos > i for pos in add_file_positions):
has_hash = _has_flag(tokens, "-hash", "--hash") has_hash = _has_flag(tokens, "-hash", "--hash")
has_store = _has_flag(tokens, "-store", "--store") has_store = _has_flag(tokens, "-instance", "--instance")
# Also accept explicit targeting via -query "store:<store> hash:<sha256> ...". # Also accept explicit targeting via -query "store:<store> hash:<sha256> ...".
query_val = _get_flag_value(tokens, "-query", "--query") query_val = _get_flag_value(tokens, "-query", "--query")
+165 -102
View File
@@ -97,16 +97,27 @@ def _log_config_load_summary(config: Dict[str, Any]) -> None:
plugin_block = config.get("plugin") plugin_block = config.get("plugin")
if not isinstance(plugin_block, dict): if not isinstance(plugin_block, dict):
plugin_block = config.get("provider") plugin_block = config.get("provider")
provs = list(plugin_block.keys()) if isinstance(plugin_block, dict) else [] if isinstance(plugin_block, dict):
stores = list(config.get("store", {}).keys()) if isinstance(config.get("store"), dict) else [] # Count distinct plugin names; note multi-instance plugins appear once per name
plugin_names = list(plugin_block.keys())
# Count total configured instances across all plugins
total_instances = sum(
len(v) if isinstance(v, dict) and all(isinstance(x, dict) for x in v.values()) else 1
for v in plugin_block.values()
if isinstance(v, dict)
)
else:
plugin_names, total_instances = [], 0
mtime = None mtime = None
try: try:
mtime = datetime.datetime.fromtimestamp(db.db_path.stat().st_mtime, datetime.timezone.utc).isoformat().replace('+00:00', 'Z') mtime = datetime.datetime.fromtimestamp(db.db_path.stat().st_mtime, datetime.timezone.utc).isoformat().replace('+00:00', 'Z')
except Exception: except Exception:
mtime = None mtime = None
plugins_str = ', '.join(plugin_names[:10]) + ('...' if len(plugin_names) > 10 else '')
summary = ( summary = (
f"Loaded config from {db.db_path.name}: plugins={len(provs)} ({', '.join(provs[:10])}{'...' if len(provs)>10 else ''}), " f"Loaded config from {db.db_path.name}: "
f"stores={len(stores)} ({', '.join(stores[:10])}{'...' if len(stores)>10 else ''}), mtime={mtime}" f"plugins={len(plugin_names)} ({plugins_str}), "
f"instances={total_instances}, mtime={mtime}"
) )
log(summary) log(summary)
except Exception: except Exception:
@@ -254,37 +265,37 @@ def set_nested_config_value(
def get_hydrus_instance( def get_hydrus_instance(
config: Dict[str, Any], instance_name: str = "home" config: Dict[str, Any], instance_name: str = "home"
) -> Optional[Dict[str, Any]]: ) -> Optional[Dict[str, Any]]:
"""Get a specific Hydrus instance config by name. """Get a specific Hydrus instance config by name from plugin/provider config."""
def _lookup_in(source: Dict[str, Any]) -> Optional[Dict[str, Any]]:
if not isinstance(source, dict) or not source:
return None
instance = source.get(instance_name)
if isinstance(instance, dict):
return instance
target = str(instance_name or "").lower()
for name, conf in source.items():
if isinstance(conf, dict) and str(name).lower() == target:
return conf
keys = sorted(source.keys())
for key in keys:
if not str(key or "").startswith("new_"):
candidate = source.get(key)
if isinstance(candidate, dict):
return candidate
first_key = keys[0] if keys else None
candidate = source.get(first_key) if first_key else None
return candidate if isinstance(candidate, dict) else None
Supports modern config plus a fallback when no exact match exists. # New format: config["plugin"]["hydrusnetwork"] or config["provider"]["hydrusnetwork"]
""" # (both point to the same dict after normalization)
store = config.get("store", {}) for section in ("plugin", "provider"):
if not isinstance(store, dict): section_cfg = config.get(section)
return None if isinstance(section_cfg, dict):
hydrus_cfg = section_cfg.get("hydrusnetwork")
hydrusnetwork = store.get("hydrusnetwork", {}) if isinstance(hydrus_cfg, dict):
if not isinstance(hydrusnetwork, dict) or not hydrusnetwork: result = _lookup_in(hydrus_cfg)
return None if result is not None:
return result
instance = hydrusnetwork.get(instance_name)
if isinstance(instance, dict):
return instance
target = str(instance_name or "").lower()
for name, conf in hydrusnetwork.items():
if isinstance(conf, dict) and str(name).lower() == target:
return conf
keys = sorted(hydrusnetwork.keys())
for key in keys:
if not str(key or "").startswith("new_"):
candidate = hydrusnetwork.get(key)
if isinstance(candidate, dict):
return candidate
first_key = keys[0]
candidate = hydrusnetwork.get(first_key)
if isinstance(candidate, dict):
return candidate
return None return None
@@ -293,7 +304,7 @@ def get_hydrus_access_key(config: Dict[str, Any], instance_name: str = "home") -
"""Get Hydrus access key for an instance. """Get Hydrus access key for an instance.
Config format: Config format:
- config["store"]["hydrusnetwork"][name]["API"] - config["plugin"]["hydrusnetwork"][name]["API"]
Args: Args:
config: Configuration dict config: Configuration dict
@@ -314,7 +325,7 @@ def get_hydrus_url(config: Dict[str, Any], instance_name: str = "home") -> Optio
"""Get Hydrus URL for an instance. """Get Hydrus URL for an instance.
Config format: Config format:
- config["store"]["hydrusnetwork"][name]["URL"] - config["plugin"]["hydrusnetwork"][name]["URL"]
Args: Args:
config: Configuration dict config: Configuration dict
@@ -438,9 +449,7 @@ def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]:
def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> Optional[str]: def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> Optional[str]:
"""Get Debrid API key from config. """Get Debrid API key from config.
Config format: Checks the plugin/provider block first (canonical format).
- config["store"]["debrid"][<name>]["api_key"]
where <name> is the store name (e.g. "all-debrid")
Args: Args:
config: Configuration dict config: Configuration dict
@@ -449,23 +458,27 @@ def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> O
Returns: Returns:
API key string if found, None otherwise API key string if found, None otherwise
""" """
store = config.get("store", {}) # 1) Canonical plugin/provider block: config["plugin"]["alldebrid"]["api_key"]
if not isinstance(store, dict): provider_block = config.get("provider") or config.get("plugin")
return None if isinstance(provider_block, dict):
alldebrid_entry = provider_block.get("alldebrid")
if isinstance(alldebrid_entry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
val = alldebrid_entry.get(k)
if isinstance(val, str) and val.strip():
return val.strip()
debrid_config = store.get("debrid", {}) # 2) Migrated legacy debrid plugin entry: config["plugin"]["debrid"]["all-debrid"]["api_key"]
if not isinstance(debrid_config, dict): if isinstance(provider_block, dict):
return None service_key = str(service).strip().lower()
debrid_plugin = provider_block.get("debrid")
service_key = str(service).strip().lower() if isinstance(debrid_plugin, dict):
entry = debrid_config.get(service_key) entry = debrid_plugin.get(service_key)
if isinstance(entry, dict):
if isinstance(entry, dict): api_key = entry.get("api_key")
api_key = entry.get("api_key") return str(api_key).strip() if api_key else None
return str(api_key).strip() if api_key else None if isinstance(entry, str):
return entry.strip() or None
if isinstance(entry, str):
return entry.strip() or None
return None return None
@@ -635,6 +648,25 @@ def _normalize_plugin_config_aliases(config: Dict[str, Any]) -> None:
if normalized_key and normalized_key not in normalized_provider: if normalized_key and normalized_key not in normalized_provider:
normalized_provider[normalized_key] = value normalized_provider[normalized_key] = value
# Fold legacy config["store"] entries into the plugin namespace.
# store format: {type: {instance_name: {key: val}}} — multi-instance.
# After folding, remove config["store"] so it is no longer consulted.
store_block = config.pop("store", None)
if isinstance(store_block, dict):
for store_type, instances in store_block.items():
if not isinstance(instances, dict):
continue
normalized_key = _normalize_provider_name(store_type)
if not normalized_key:
continue
existing = normalized_provider.get(normalized_key)
if not isinstance(existing, dict):
existing = {}
normalized_provider[normalized_key] = existing
for instance_name, settings in instances.items():
if isinstance(settings, dict) and instance_name not in existing:
existing[instance_name] = dict(settings)
if normalized_provider: if normalized_provider:
config["provider"] = normalized_provider config["provider"] = normalized_provider
config["plugin"] = normalized_provider config["plugin"] = normalized_provider
@@ -658,6 +690,11 @@ def _extract_api_key(value: Any) -> Optional[str]:
def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None: def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None:
"""Ensure AllDebrid API key is consistently stored in config[\"plugin\"][\"alldebrid\"].
Previously this function also synced to config[\"store\"][\"debrid\"]. That path
is no longer used; only the plugin namespace is written.
"""
if not isinstance(config, dict): if not isinstance(config, dict):
return return
@@ -680,38 +717,39 @@ def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None:
provider_section = {"api_key": provider_key} provider_section = {"api_key": provider_key}
providers["alldebrid"] = provider_section providers["alldebrid"] = provider_section
store_block = config.get("store") # If no key found in provider block, check for a migrated debrid plugin entry.
if not isinstance(store_block, dict): # (rows_to_config migrates store.debrid.all-debrid → plugin.debrid.all-debrid)
store_block = {} if not provider_key:
config["store"] = store_block plugin_block = config.get("plugin") or providers
debrid_plugin = plugin_block.get("debrid") if isinstance(plugin_block, dict) else None
if isinstance(debrid_plugin, dict):
service_entry = debrid_plugin.get("all-debrid")
legacy_key = _extract_api_key(service_entry) if service_entry else None
if legacy_key:
if provider_section is None:
provider_section = {}
providers["alldebrid"] = provider_section
provider_section.setdefault("api_key", legacy_key)
debrid_block = store_block.get("debrid")
store_key = None
if isinstance(debrid_block, dict):
service_entry = debrid_block.get("all-debrid")
if isinstance(service_entry, dict):
store_key = _extract_api_key(service_entry)
elif isinstance(service_entry, str):
store_key = service_entry.strip()
if store_key:
debrid_block["all-debrid"] = {"api_key": store_key}
else:
debrid_block = None
if provider_key:
if debrid_block is None: def _is_multi_instance_plugin_config(value: Any) -> bool:
debrid_block = {} """Return True if `value` looks like a multi-instance plugin config (dict-of-dicts).
store_block["debrid"] = debrid_block
service_section = debrid_block.get("all-debrid") Multi-instance plugins store their configuration as::
if not isinstance(service_section, dict):
service_section = {} {<instance_name>: {key: value, ...}, ...}
debrid_block["all-debrid"] = service_section
service_section["api_key"] = provider_key Single-instance plugins store their config as a flat dict::
elif store_key:
if provider_section is None: {key: value, ...}
provider_section = {}
providers["alldebrid"] = provider_section We detect multi-instance by checking whether ALL values are themselves dicts
provider_section["api_key"] = store_key (and the outer dict is non-empty). An empty dict is treated as single-instance.
"""
if not isinstance(value, dict) or not value:
return False
return all(isinstance(v, dict) for v in value.values())
def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str, str], Any]: def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str, str], Any]:
@@ -719,18 +757,35 @@ def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str,
_normalize_plugin_config_aliases(config) _normalize_plugin_config_aliases(config)
for key, value in config.items(): for key, value in config.items():
if key == 'plugin': if key == 'plugin':
# plugin == provider after normalization; skip duplicate
continue continue
if key in ('store', 'provider', 'tool') and isinstance(value, dict): if key == 'provider' and isinstance(value, dict):
for subtype, plugin_cfg in value.items():
if not isinstance(plugin_cfg, dict):
continue
if _is_multi_instance_plugin_config(plugin_cfg):
# Multi-instance: {instance_name: {key: val}}
for instance_name, settings in plugin_cfg.items():
if not isinstance(settings, dict):
continue
for k, v in settings.items():
entries[('plugin', subtype, instance_name, k)] = v
else:
# Single-instance: {key: val}
for k, v in plugin_cfg.items():
entries[('provider', subtype, 'default', k)] = v
elif key in ('store', 'tool') and isinstance(value, dict):
for subtype, instances in value.items(): for subtype, instances in value.items():
if not isinstance(instances, dict): if not isinstance(instances, dict):
continue continue
if key == 'store': if key == 'store':
# Legacy store: migrate to plugin category
for name, settings in instances.items(): for name, settings in instances.items():
if not isinstance(settings, dict): if not isinstance(settings, dict):
continue continue
for k, v in settings.items(): for k, v in settings.items():
entries[(key, subtype, name, k)] = v entries[('plugin', subtype, name, k)] = v
else: else: # tool
for k, v in instances.items(): for k, v in instances.items():
entries[(key, subtype, 'default', k)] = v entries[(key, subtype, 'default', k)] = v
elif not key.startswith('_') and value is not None: elif not key.startswith('_') and value is not None:
@@ -763,12 +818,23 @@ def _config_from_flattened_entries(
continue continue
if category == "store": if category == "store":
store_block = config.setdefault("store", {}) # Legacy: migrate to plugin namespace at reconstitution time
subtype_block = store_block.setdefault(subtype, {}) plugin_block = config.setdefault("plugin", {})
subtype_block = plugin_block.setdefault(subtype, {})
item_block = subtype_block.setdefault(item_name, {}) item_block = subtype_block.setdefault(item_name, {})
item_block[key] = value item_block[key] = value
continue continue
if category == "plugin":
plugin_block = config.setdefault("plugin", {})
subtype_block = plugin_block.setdefault(subtype, {})
if item_name == "default":
subtype_block[key] = value
else:
item_block = subtype_block.setdefault(item_name, {})
item_block[key] = value
continue
if category in {"provider", "tool"}: if category in {"provider", "tool"}:
category_block = config.setdefault(category, {}) category_block = config.setdefault(category, {})
subtype_block = category_block.setdefault(subtype, {}) subtype_block = category_block.setdefault(subtype, {})
@@ -827,18 +893,7 @@ def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]:
elif isinstance(entry, str) and entry.strip(): elif isinstance(entry, str) and entry.strip():
expected_key = entry.strip() expected_key = entry.strip()
if not expected_key: if not expected_key:
store_block = config.get("store", {}) if isinstance(config, dict) else {} expected_key = get_debrid_api_key(config, service="All-debrid")
debrid = store_block.get("debrid") if isinstance(store_block, dict) else None
if isinstance(debrid, dict):
srv = debrid.get("all-debrid")
if isinstance(srv, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = srv.get(k)
if isinstance(v, str) and v.strip():
expected_key = v.strip()
break
elif isinstance(srv, str) and srv.strip():
expected_key = srv.strip()
except Exception as exc: except Exception as exc:
logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True) logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True)
expected_key = None expected_key = None
@@ -853,6 +908,14 @@ def load_config(*, emit_summary: bool = True) -> Dict[str, Any]:
_CONFIG_SUMMARY_PENDING = False _CONFIG_SUMMARY_PENDING = False
return _CONFIG_CACHE return _CONFIG_CACHE
# One-time DB migration: move category='store' rows to category='plugin'.
# This is idempotent — a no-op if no store rows exist.
try:
from SYS.database import migrate_store_category_to_plugin
migrate_store_category_to_plugin()
except Exception:
logger.debug("Store→plugin DB migration skipped or failed", exc_info=True)
# Load strictly from database # Load strictly from database
db_config = get_config_all() db_config = get_config_all()
if db_config: if db_config:
+45 -1
View File
@@ -510,7 +510,10 @@ def rows_to_config(rows) -> Dict[str, Any]:
sub_dict = cat_dict.setdefault(sub, {}) sub_dict = cat_dict.setdefault(sub, {})
sub_dict[key] = parsed_val sub_dict[key] = parsed_val
elif cat == 'store': elif cat == 'store':
cat_dict = config.setdefault(cat, {}) # Migrate legacy store rows into the unified plugin namespace.
# store config used a 4-level path: (store, type, instance_name, key).
# Plugin config uses: config["plugin"][type][instance_name][key].
cat_dict = config.setdefault('plugin', {})
sub_dict = cat_dict.setdefault(sub, {}) sub_dict = cat_dict.setdefault(sub, {})
name_dict = sub_dict.setdefault(name, {}) name_dict = sub_dict.setdefault(name, {})
name_dict[key] = parsed_val name_dict[key] = parsed_val
@@ -520,11 +523,52 @@ def rows_to_config(rows) -> Dict[str, Any]:
return config return config
def migrate_store_category_to_plugin() -> int:
"""One-time migration: re-key category='store' DB rows to category='plugin'.
The 'store' category used ``(store, type, instance_name, key)`` tuples;
the unified plugin system uses the same 4-level path under category='plugin'.
Existing 'plugin' rows for the same (subtype, item_name, key) are overwritten.
Returns the number of rows that were migrated (0 if already migrated).
"""
try:
count_row = db.fetchone(
"SELECT COUNT(*) AS n FROM config WHERE category='store' AND subtype != 'folder'"
)
count = int(count_row['n']) if count_row else 0
if count == 0:
# Also clean up any lingering folder-store rows
db.execute("DELETE FROM config WHERE category='store'")
with db._conn_lock:
db.conn.commit()
return 0
# Copy store rows to plugin, replacing any pre-existing plugin rows for
# the same (subtype, item_name, key), then delete the old store rows.
db.execute(
"""
INSERT OR REPLACE INTO config (category, subtype, item_name, key, value)
SELECT 'plugin', subtype, item_name, key, value
FROM config
WHERE category = 'store' AND subtype != 'folder'
"""
)
db.execute("DELETE FROM config WHERE category = 'store'")
with db._conn_lock:
db.conn.commit()
logger.info("Migrated %d config rows from category='store' to category='plugin'", count)
return count
except Exception:
logger.exception("Failed to migrate store config rows to plugin category")
return 0
def get_config_all() -> Dict[str, Any]: def get_config_all() -> Dict[str, Any]:
"""Retrieve all configuration from the database in the legacy dict format.""" """Retrieve all configuration from the database in the legacy dict format."""
rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config") rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config")
return rows_to_config(rows) return rows_to_config(rows)
# Worker Management Methods for medios.db # Worker Management Methods for medios.db
def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection: def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection:
+2 -2
View File
@@ -1272,7 +1272,7 @@ class PipelineExecutor:
"""Guard against running add-relationship on unstored download-file results. """Guard against running add-relationship on unstored download-file results.
Intended UX: Intended UX:
download-file ... | add-file -store <store> | add-relationship download-file ... | add-file -instance <store> | add-relationship
Rationale: Rationale:
download-file outputs items that may not yet have a stable store+hash. download-file outputs items that may not yet have a stable store+hash.
@@ -1305,7 +1305,7 @@ class PipelineExecutor:
print( print(
"Pipeline order error: when using download-file with add-relationship, " "Pipeline order error: when using download-file with add-relationship, "
"add-relationship must come after add-file (so items are stored and have store+hash).\n" "add-relationship must come after add-file (so items are stored and have store+hash).\n"
"Example: download-file <...> | add-file -store <store> | add-relationship\n" "Example: download-file <...> | add-file -instance <store> | add-relationship\n"
) )
return False return False
+41 -31
View File
@@ -7,7 +7,6 @@ from typing import Any, Dict, Iterable, List, Optional
from SYS.config import global_config from SYS.config import global_config
from ProviderCore.registry import get_plugin_class, list_plugins from ProviderCore.registry import get_plugin_class, list_plugins
from Store.registry import _discover_store_classes, _required_keys_for, _resolve_store_class
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -54,10 +53,16 @@ def _call_schema(owner: Any, label: str) -> List[ConfigField]:
def get_store_schema(store_type: str) -> List[ConfigField]: def get_store_schema(store_type: str) -> List[ConfigField]:
cls = _resolve_store_class(str(store_type or "").strip()) """Return config schema for a store type.
if cls is None:
return [] After the store→plugin migration, store types are plugins. We look up the
return _call_schema(cls, f"store '{store_type}'") plugin schema by name; if not found we return an empty list.
"""
normalized = str(store_type or "").strip()
# Strip a legacy "store-" prefix so callers using the old type name still work
if normalized.startswith("store-"):
normalized = normalized[len("store-"):]
return get_plugin_schema(normalized)
def get_plugin_schema(plugin_name: str) -> List[ConfigField]: def get_plugin_schema(plugin_name: str) -> List[ConfigField]:
@@ -84,6 +89,10 @@ def get_item_schema(item_type: str, item_name: str) -> List[ConfigField]:
normalized_name = str(item_name or "").strip() normalized_name = str(item_name or "").strip()
if normalized_type.startswith("store-"): if normalized_type.startswith("store-"):
return get_store_schema(normalized_type.replace("store-", "", 1)) return get_store_schema(normalized_type.replace("store-", "", 1))
if normalized_type.startswith("plugin-"):
# Multi-instance plugin: plugin-{ptype}; item_name is the instance name
ptype = normalized_type[len("plugin-"):]
return get_plugin_schema(ptype)
if normalized_type in {"provider", "plugin"}: if normalized_type in {"provider", "plugin"}:
return get_plugin_schema(normalized_name) return get_plugin_schema(normalized_name)
if normalized_type == "tool": if normalized_type == "tool":
@@ -104,23 +113,14 @@ def get_global_schema_map() -> Dict[str, ConfigField]:
def build_default_store_config(store_type: str, instance_name: str) -> Dict[str, Any]: def build_default_store_config(store_type: str, instance_name: str) -> Dict[str, Any]:
"""Build a default config dict for a new store/multi-instance plugin entry."""
config: Dict[str, Any] = {"NAME": instance_name} config: Dict[str, Any] = {"NAME": instance_name}
schema = get_store_schema(store_type) schema = get_store_schema(store_type)
if schema: for field in schema:
for field in schema: key = field["key"]
key = field["key"] if key.upper() == "NAME":
if key.upper() == "NAME":
continue
config[key] = field.get("default", "")
return config
cls = _resolve_store_class(str(store_type or "").strip())
if cls is None:
return config
for required_key in _required_keys_for(cls):
if required_key.upper() == "NAME":
continue continue
config[required_key] = "" config[key] = field.get("default", "")
return config return config
@@ -170,12 +170,16 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]:
if field.get("required"): if field.get("required"):
_add_key(field.get("key")) _add_key(field.get("key"))
if normalized_type.startswith("store-"): if normalized_type.startswith("plugin-") or normalized_type.startswith("store-"):
store_type = normalized_type.replace("store-", "", 1) # Multi-instance plugin (plugin-{ptype}) or legacy store-{type}: look up by plugin name
cls = _resolve_store_class(store_type) ptype = normalized_type.replace("plugin-", "", 1).replace("store-", "", 1)
if cls is not None: plugin_class = get_plugin_class(ptype)
for required_key in _required_keys_for(cls): if plugin_class is not None:
_add_key(required_key) try:
for required_key in plugin_class.required_config_keys():
_add_key(required_key)
except Exception:
logger.exception("Failed to load required config keys for plugin '%s'", ptype)
elif normalized_type in {"provider", "plugin"}: elif normalized_type in {"provider", "plugin"}:
plugin_class = get_plugin_class(normalized_name) plugin_class = get_plugin_class(normalized_name)
if plugin_class is not None: if plugin_class is not None:
@@ -189,18 +193,24 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]:
def get_configurable_store_types() -> List[str]: def get_configurable_store_types() -> List[str]:
"""Return configurable multi-instance plugin types (formerly 'store types')."""
from ProviderCore.registry import REGISTRY
options: List[str] = [] options: List[str] = []
for store_type in _discover_store_classes().keys(): for info in REGISTRY.iter_plugins():
if get_store_schema(store_type): plugin_cls = info.plugin_class
options.append(str(store_type)) if getattr(plugin_cls, 'MULTI_INSTANCE', False) and get_plugin_schema(info.canonical_name):
options.append(info.canonical_name)
return sorted(set(options)) return sorted(set(options))
def get_configurable_plugin_types() -> List[str]: def get_configurable_plugin_types() -> List[str]:
"""Return all plugin types that can be configured: those with a schema or MULTI_INSTANCE flag."""
from ProviderCore.registry import REGISTRY
options: List[str] = [] options: List[str] = []
for plugin_name in list_plugins().keys(): for info in REGISTRY.iter_plugins():
if get_plugin_schema(plugin_name): plugin_cls = info.plugin_class
options.append(str(plugin_name)) if get_plugin_schema(info.canonical_name) or getattr(plugin_cls, 'MULTI_INSTANCE', False):
options.append(info.canonical_name)
return sorted(set(options)) return sorted(set(options))
+6
View File
@@ -92,6 +92,8 @@ def show_plugin_config_panel(
"""Show a Rich panel explaining how to configure plugins.""" """Show a Rich panel explaining how to configure plugins."""
from rich.table import Table as RichTable from rich.table import Table as RichTable
from rich.console import Group from rich.console import Group
from rich.panel import Panel
from rich.text import Text
if isinstance(plugin_names, str): if isinstance(plugin_names, str):
plugins = [p.strip() for p in plugin_names.split(",")] plugins = [p.strip() for p in plugin_names.split(",")]
@@ -127,6 +129,8 @@ def show_store_config_panel(
"""Show a Rich panel explaining how to configure storage backends.""" """Show a Rich panel explaining how to configure storage backends."""
from rich.table import Table as RichTable from rich.table import Table as RichTable
from rich.console import Group from rich.console import Group
from rich.panel import Panel
from rich.text import Text
if isinstance(store_names, str): if isinstance(store_names, str):
stores = [s.strip() for s in store_names.split(",")] stores = [s.strip() for s in store_names.split(",")]
@@ -160,6 +164,8 @@ def show_available_plugins_panel(plugin_names: List[str]) -> None:
"""Show a Rich panel listing available/configured plugins.""" """Show a Rich panel listing available/configured plugins."""
from rich.columns import Columns from rich.columns import Columns
from rich.console import Group from rich.console import Group
from rich.panel import Panel
from rich.text import Text
if not plugin_names: if not plugin_names:
return return
+1 -1
View File
@@ -45,7 +45,7 @@ def build_hash_store_selection(
store_text = str(store_value or "").strip() store_text = str(store_value or "").strip()
if not hash_text or not store_text: if not hash_text or not store_text:
return None, None return None, None
args = ["-query", f"hash:{hash_text}", "-store", store_text] args = ["-query", f"hash:{hash_text}", "-instance", store_text]
return args, [action_name] + list(args) return args, [action_name] + list(args)
+10 -10
View File
@@ -364,7 +364,7 @@ class TagEditorPopup(ModalScreen[None]):
if to_del: if to_del:
del_args = " ".join(json.dumps(t) for t in to_del) del_args = " ".join(json.dumps(t) for t in to_del)
del_cmd = f"delete-tag -store {store_tok}{query_chunk} {del_args}" del_cmd = f"delete-tag -instance {store_tok}{query_chunk} {del_args}"
_log_pipeline_command("delete-tag", del_cmd) _log_pipeline_command("delete-tag", del_cmd)
del_res = runner.run_pipeline(del_cmd, seeds=self._seeds, isolate=True) del_res = runner.run_pipeline(del_cmd, seeds=self._seeds, isolate=True)
_log_pipeline_result("delete-tag", del_res) _log_pipeline_result("delete-tag", del_res)
@@ -381,7 +381,7 @@ class TagEditorPopup(ModalScreen[None]):
if to_add: if to_add:
add_args = " ".join(json.dumps(t) for t in to_add) add_args = " ".join(json.dumps(t) for t in to_add)
add_cmd = f"add-tag -store {store_tok}{query_chunk} {add_args}" add_cmd = f"add-tag -instance {store_tok}{query_chunk} {add_args}"
_log_pipeline_command("add-tag", add_cmd) _log_pipeline_command("add-tag", add_cmd)
add_res = runner.run_pipeline(add_cmd, seeds=self._seeds, isolate=True) add_res = runner.run_pipeline(add_cmd, seeds=self._seeds, isolate=True)
_log_pipeline_result("add-tag", add_res) _log_pipeline_result("add-tag", add_res)
@@ -1028,7 +1028,7 @@ class PipelineHubApp(App):
Rules (simple + non-destructive): Rules (simple + non-destructive):
- If output path is set and the first stage is download-file and has no -path/--path, append -path. - If output path is set and the first stage is download-file and has no -path/--path, append -path.
- If a store is selected and pipeline has no add-file stage, append add-file -store <store>. - If an instance is selected and pipeline has no add-file stage, append add-file -instance <name>.
""" """
base = str(pipeline_text or "").strip() base = str(pipeline_text or "").strip()
if not base: if not base:
@@ -1080,7 +1080,7 @@ class PipelineHubApp(App):
if should_auto_add_file: if should_auto_add_file:
store_token = json.dumps(selected_store) store_token = json.dumps(selected_store)
joined = f"{joined} | add-file -store {store_token}" joined = f"{joined} | add-file -instance {store_token}"
return joined return joined
@@ -1656,7 +1656,7 @@ class PipelineHubApp(App):
try: try:
if to_del: if to_del:
del_args = " ".join(json.dumps(t) for t in to_del) del_args = " ".join(json.dumps(t) for t in to_del)
del_cmd = f"delete-tag -store {store_tok}{query_chunk} {del_args}" del_cmd = f"delete-tag -instance {store_tok}{query_chunk} {del_args}"
del_res = runner.run_pipeline(del_cmd, seeds=seeds, isolate=True) del_res = runner.run_pipeline(del_cmd, seeds=seeds, isolate=True)
if not getattr(del_res, "success", False): if not getattr(del_res, "success", False):
failures.append( failures.append(
@@ -1669,7 +1669,7 @@ class PipelineHubApp(App):
if to_add: if to_add:
add_args = " ".join(json.dumps(t) for t in to_add) add_args = " ".join(json.dumps(t) for t in to_add)
add_cmd = f"add-tag -store {store_tok}{query_chunk} {add_args}" add_cmd = f"add-tag -instance {store_tok}{query_chunk} {add_args}"
add_res = runner.run_pipeline(add_cmd, seeds=seeds, isolate=True) add_res = runner.run_pipeline(add_cmd, seeds=seeds, isolate=True)
if not getattr(add_res, "success", False): if not getattr(add_res, "success", False):
failures.append( failures.append(
@@ -2358,7 +2358,7 @@ class PipelineHubApp(App):
self.notify("Delete action requires store + hash", severity="warning", timeout=3) self.notify("Delete action requires store + hash", severity="warning", timeout=3)
return return
query = f"hash:{hash_value}" query = f"hash:{hash_value}"
cmd = f"delete-file -store {json.dumps(store_name)} -query {json.dumps(query)}" cmd = f"delete-file -instance {json.dumps(store_name)} -query {json.dumps(query)}"
self._start_pipeline_execution(cmd) self._start_pipeline_execution(cmd)
return return
@@ -2398,11 +2398,11 @@ class PipelineHubApp(App):
query = f"hash:{hash_value}" query = f"hash:{hash_value}"
base_copy = ( base_copy = (
f"search-file -store {json.dumps(store_name)} {json.dumps(query)}" f"search-file -instance {json.dumps(store_name)} {json.dumps(query)}"
f" | add-file -store {json.dumps(selected_store)}" f" | add-file -instance {json.dumps(selected_store)}"
) )
if action == "move_to_selected_store": if action == "move_to_selected_store":
delete_cmd = f"delete-file -store {json.dumps(store_name)} -query {json.dumps(query)}" delete_cmd = f"delete-file -instance {json.dumps(store_name)} -query {json.dumps(query)}"
cmd = f"{base_copy} | @ | {delete_cmd}" cmd = f"{base_copy} | @ | {delete_cmd}"
else: else:
cmd = base_copy cmd = base_copy
+2 -2
View File
@@ -32,13 +32,13 @@ PIPELINE_PRESETS: List[PipelinePreset] = [
description= description=
"Use download-file with playlist auto-selection, merge the pieces, tag, then import into local storage.", "Use download-file with playlist auto-selection, merge the pieces, tag, then import into local storage.",
pipeline= pipeline=
'download-file "<url>" | merge-file | add-tags -store local | add-file -storage local', 'download-file "<url>" | merge-file | add-tags -instance local | add-file -storage local',
), ),
PipelinePreset( PipelinePreset(
label="Download → Hydrus", label="Download → Hydrus",
description="Fetch media, auto-tag, and push directly into Hydrus.", description="Fetch media, auto-tag, and push directly into Hydrus.",
pipeline= pipeline=
'download-file "<url>" | merge-file | add-tags -store hydrus | add-file -storage hydrus', 'download-file "<url>" | merge-file | add-tags -instance hydrus | add-file -storage hydrus',
), ),
PipelinePreset( PipelinePreset(
label="Search Local Library", label="Search Local Library",
+111 -90
View File
@@ -18,6 +18,7 @@ from SYS.config import (
count_changed_entries, count_changed_entries,
ConfigSaveConflict, ConfigSaveConflict,
coerce_config_value, coerce_config_value,
_is_multi_instance_plugin_config,
) )
from SYS.database import db from SYS.database import db
from SYS.logger import log, debug from SYS.logger import log, debug
@@ -200,7 +201,6 @@ class ConfigModal(ModalScreen):
yield Label("Categories", classes="config-label") yield Label("Categories", classes="config-label")
with ListView(id="category-list"): with ListView(id="category-list"):
yield ListItem(Label("Global Settings"), id="cat-globals") yield ListItem(Label("Global Settings"), id="cat-globals")
yield ListItem(Label("Stores"), id="cat-stores")
yield ListItem(Label("Plugins"), id="cat-providers") yield ListItem(Label("Plugins"), id="cat-providers")
yield ListItem(Label("Tools"), id="cat-tools") yield ListItem(Label("Tools"), id="cat-tools")
@@ -210,14 +210,12 @@ class ConfigModal(ModalScreen):
yield Button("Save", variant="success", id="save-btn") yield Button("Save", variant="success", id="save-btn")
# Durable synchronous save: waits and verifies DB persisted critical keys # Durable synchronous save: waits and verifies DB persisted critical keys
yield Button("Save (durable)", variant="primary", id="save-durable-btn") yield Button("Save (durable)", variant="primary", id="save-durable-btn")
yield Button("Add Store", variant="primary", id="add-store-btn")
yield Button("Add Plugin", variant="primary", id="add-provider-btn") yield Button("Add Plugin", variant="primary", id="add-provider-btn")
yield Button("Add Tool", variant="primary", id="add-tool-btn") yield Button("Add Tool", variant="primary", id="add-tool-btn")
yield Button("Back", id="back-btn") yield Button("Back", id="back-btn")
yield Button("Close", variant="error", id="cancel-btn") yield Button("Close", variant="error", id="cancel-btn")
def on_mount(self) -> None: def on_mount(self) -> None:
self.query_one("#add-store-btn", Button).display = False
self.query_one("#add-provider-btn", Button).display = False self.query_one("#add-provider-btn", Button).display = False
try: try:
self.query_one("#add-tool-btn", Button).display = False self.query_one("#add-tool-btn", Button).display = False
@@ -267,7 +265,6 @@ class ConfigModal(ModalScreen):
# Update visibility of buttons # Update visibility of buttons
try: try:
self.query_one("#add-store-btn", Button).display = (self.current_category == "stores" and self.editing_item_name is None)
self.query_one("#add-provider-btn", Button).display = (self.current_category == "providers" and self.editing_item_name is None) self.query_one("#add-provider-btn", Button).display = (self.current_category == "providers" and self.editing_item_name is None)
self.query_one("#add-tool-btn", Button).display = (self.current_category == "tools" and self.editing_item_name is None) self.query_one("#add-tool-btn", Button).display = (self.current_category == "tools" and self.editing_item_name is None)
self.query_one("#back-btn", Button).display = (self.editing_item_name is not None) self.query_one("#back-btn", Button).display = (self.editing_item_name is not None)
@@ -467,20 +464,46 @@ class ConfigModal(ModalScreen):
providers = self.config_data.get("provider", {}) providers = self.config_data.get("provider", {})
if not providers: if not providers:
container.mount(Static("No plugins configured.")) container.mount(Static("No plugins configured."))
else: return
for i, (name, _) in enumerate(providers.items()):
edit_id = f"edit-provider-{i}"
del_id = f"del-provider-{i}"
self._button_id_map[edit_id] = ("edit", "provider", name)
self._button_id_map[del_id] = ("del", "provider", name)
idx = 0
for plugin_name, plugin_cfg in providers.items():
if isinstance(plugin_cfg, dict) and _is_multi_instance_plugin_config(plugin_cfg):
# Multi-instance plugin: show each instance as a separate row
for instance_name, instance_cfg in plugin_cfg.items():
display_name = instance_name
if isinstance(instance_cfg, dict):
display_name = (
instance_cfg.get("NAME")
or instance_cfg.get("name")
or instance_name
)
edit_id = f"edit-provider-{idx}"
del_id = f"del-provider-{idx}"
self._button_id_map[edit_id] = ("edit", f"plugin-{plugin_name}", instance_name)
self._button_id_map[del_id] = ("del", f"plugin-{plugin_name}", instance_name)
row = Horizontal(
Static(f"{display_name} ({plugin_name})", classes="item-label"),
Button("Edit", id=edit_id),
Button("Delete", variant="error", id=del_id),
classes="item-row"
)
container.mount(row)
idx += 1
else:
# Single-instance plugin
edit_id = f"edit-provider-{idx}"
del_id = f"del-provider-{idx}"
self._button_id_map[edit_id] = ("edit", "plugin", plugin_name)
self._button_id_map[del_id] = ("del", "plugin", plugin_name)
row = Horizontal( row = Horizontal(
Static(name, classes="item-label"), Static(plugin_name, classes="item-label"),
Button("Edit", id=edit_id), Button("Edit", id=edit_id),
Button("Delete", variant="error", id=del_id), Button("Delete", variant="error", id=del_id),
classes="item-row" classes="item-row"
) )
container.mount(row) container.mount(row)
idx += 1
def render_tools(self, container: ScrollableContainer) -> None: def render_tools(self, container: ScrollableContainer) -> None:
container.mount(Label("Configured Tools", classes="config-label")) container.mount(Label("Configured Tools", classes="config-label"))
@@ -508,11 +531,13 @@ class ConfigModal(ModalScreen):
item_schema_map = get_item_schema_map(item_type, item_name) item_schema_map = get_item_schema_map(item_type, item_name)
render_state = {"group": None, "mounted_any": False} render_state = {"group": None, "mounted_any": False}
# Parse item_type for store-{stype} or just provider # Parse item_type: plugin-{ptype} (multi-instance) or flat type
if item_type.startswith("store-"): if item_type.startswith("plugin-"):
stype = item_type.replace("store-", "") ptype = item_type[len("plugin-"):]
container.mount(Label(f"Editing Store: {item_name} ({stype})", classes="config-label")) container.mount(Label(f"Editing {ptype}: {item_name}", classes="config-label"))
section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {}) plugin_block = self.config_data.get("plugin") or self.config_data.get("provider") or {}
plugin_instances = plugin_block.get(ptype, {}) if isinstance(plugin_block, dict) else {}
section = plugin_instances.get(item_name, {}) if isinstance(plugin_instances, dict) else {}
else: else:
container.mount(Label(f"Editing {item_type.capitalize()}: {item_name}", classes="config-label")) container.mount(Label(f"Editing {item_type.capitalize()}: {item_name}", classes="config-label"))
section = self.config_data.get(item_type, {}).get(item_name, {}) section = self.config_data.get(item_type, {}).get(item_name, {})
@@ -598,7 +623,7 @@ class ConfigModal(ModalScreen):
row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn"))
idx += 1 idx += 1
if item_type == "plugin" and isinstance(item_name, str): if item_type in ("plugin", "provider") and isinstance(item_name, str):
provider = self._instantiate_plugin_for_editor(item_name, self.config_data) provider = self._instantiate_plugin_for_editor(item_name, self.config_data)
if provider is not None: if provider is not None:
provider_actions = provider.config_actions() or [] provider_actions = provider.config_actions() or []
@@ -626,7 +651,7 @@ class ConfigModal(ModalScreen):
) )
if ( if (
item_type == "plugin" item_type in ("plugin", "provider")
and isinstance(item_name, str) and isinstance(item_name, str)
and item_name.strip().lower() == "matrix" and item_name.strip().lower() == "matrix"
): ):
@@ -720,13 +745,11 @@ class ConfigModal(ModalScreen):
if not event.item: if not event.item:
return return
item_id = getattr(event.item, "id", None) item_id = getattr(event.item, "id", None)
if item_id not in ("cat-globals", "cat-stores", "cat-providers", "cat-tools"): if item_id not in ("cat-globals", "cat-providers", "cat-tools"):
return return
if item_id == "cat-globals": if item_id == "cat-globals":
self.current_category = "globals" self.current_category = "globals"
elif item_id == "cat-stores":
self.current_category = "stores"
elif item_id == "cat-providers": elif item_id == "cat-providers":
self.current_category = "providers" self.current_category = "providers"
elif item_id == "cat-tools": elif item_id == "cat-tools":
@@ -841,13 +864,23 @@ class ConfigModal(ModalScreen):
self.refresh_view() self.refresh_view()
elif action == "del": elif action == "del":
removed = False removed = False
if itype.startswith("store-"): if itype.startswith("plugin-"):
ptype = itype[len("plugin-"):]
plugin_block = self.config_data.get("plugin") or self.config_data.get("provider")
if isinstance(plugin_block, dict):
instances = plugin_block.get(ptype)
if isinstance(instances, dict) and name in instances:
del instances[name]
if not instances:
plugin_block.pop(ptype, None)
removed = True
elif itype.startswith("store-"):
stype = itype.replace("store-", "") stype = itype.replace("store-", "")
if "store" in self.config_data and stype in self.config_data["store"]: if "store" in self.config_data and stype in self.config_data["store"]:
if name in self.config_data["store"][stype]: if name in self.config_data["store"][stype]:
del self.config_data["store"][stype][name] del self.config_data["store"][stype][name]
removed = True removed = True
elif itype == "provider": elif itype in ("provider", "plugin"):
if "provider" in self.config_data and name in self.config_data["provider"]: if "provider" in self.config_data and name in self.config_data["provider"]:
del self.config_data["provider"][name] del self.config_data["provider"][name]
removed = True removed = True
@@ -871,9 +904,6 @@ class ConfigModal(ModalScreen):
elif bid in self._provider_button_map: elif bid in self._provider_button_map:
provider_name, action_id = self._provider_button_map[bid] provider_name, action_id = self._provider_button_map[bid]
self._request_plugin_action(provider_name, action_id) self._request_plugin_action(provider_name, action_id)
elif bid == "add-store-btn":
options = get_configurable_store_types()
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
elif bid == "add-provider-btn": elif bid == "add-provider-btn":
options = get_configurable_plugin_types() options = get_configurable_plugin_types()
self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected) self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected)
@@ -1036,51 +1066,44 @@ class ConfigModal(ModalScreen):
# Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed. # Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed.
def on_store_type_selected(self, stype: str) -> None:
if not stype:
return
self._capture_editor_snapshot()
existing_names: set[str] = set()
store_block = self.config_data.get("store")
if isinstance(store_block, dict):
st_entries = store_block.get(stype)
if isinstance(st_entries, dict):
existing_names = {str(name) for name in st_entries.keys() if name}
base_name = f"new_{stype}"
new_name = base_name
suffix = 1
while new_name in existing_names:
suffix += 1
new_name = f"{base_name}_{suffix}"
if "store" not in self.config_data:
self.config_data["store"] = {}
if stype not in self.config_data["store"]:
self.config_data["store"][stype] = {}
# Default config for the new store
new_config = build_default_store_config(stype, new_name)
self.config_data["store"][stype][new_name] = new_config
self.editing_item_type = f"store-{stype}"
self.editing_item_name = new_name
self.refresh_view()
def on_provider_type_selected(self, ptype: str) -> None: def on_provider_type_selected(self, ptype: str) -> None:
if not ptype: return if not ptype:
return
self._capture_editor_snapshot() self._capture_editor_snapshot()
if "provider" not in self.config_data:
self.config_data["provider"] = {}
# Plugins are configured under the top-level 'provider' dict for now. from ProviderCore.registry import get_plugin_class as _get_cls
if ptype not in self.config_data["provider"]: plugin_class = _get_cls(ptype)
self.config_data["provider"][ptype] = build_default_plugin_config(ptype) is_multi = bool(getattr(plugin_class, 'MULTI_INSTANCE', False)) if plugin_class else False
if is_multi:
# Multi-instance plugin: create a named instance entry in config["plugin"][ptype]
plugin_block = self.config_data.setdefault("plugin", {})
instances = plugin_block.setdefault(ptype, {})
# Also keep config["provider"] in sync (they should be the same dict after normalization,
# but if they're not yet, link them)
if "provider" in self.config_data and self.config_data["provider"] is not plugin_block:
self.config_data["provider"].setdefault(ptype, instances)
existing_names: set[str] = set(instances.keys())
base_name = f"new_{ptype}"
new_name = base_name
suffix = 1
while new_name in existing_names:
suffix += 1
new_name = f"{base_name}_{suffix}"
instances[new_name] = build_default_store_config(ptype, new_name)
self.editing_item_type = f"plugin-{ptype}"
self.editing_item_name = new_name
else:
# Single-instance plugin
if "provider" not in self.config_data:
self.config_data["provider"] = {}
if ptype not in self.config_data["provider"]:
self.config_data["provider"][ptype] = build_default_plugin_config(ptype)
self.editing_item_type = "plugin"
self.editing_item_name = ptype
self.editing_item_type = "plugin"
self.editing_item_name = ptype
self.refresh_view() self.refresh_view()
def on_tool_type_selected(self, tname: str) -> None: def on_tool_type_selected(self, tname: str) -> None:
@@ -1112,13 +1135,13 @@ class ConfigModal(ModalScreen):
if widget_id.startswith("global-"): if widget_id.startswith("global-"):
existing_value = self.config_data.get(key) existing_value = self.config_data.get(key)
elif widget_id.startswith("item-") and item_name: elif widget_id.startswith("item-") and item_name:
if item_type.startswith("store-"): if item_type.startswith("plugin-"):
stype = item_type.replace("store-", "") ptype = item_type[len("plugin-"):]
store_block = self.config_data.get("store") plugin_block = self.config_data.get("plugin") or self.config_data.get("provider")
if isinstance(store_block, dict): if isinstance(plugin_block, dict):
type_block = store_block.get(stype) instances = plugin_block.get(ptype)
if isinstance(type_block, dict): if isinstance(instances, dict):
section = type_block.get(item_name) section = instances.get(item_name)
if isinstance(section, dict): if isinstance(section, dict):
existing_value = section.get(key) existing_value = section.get(key)
else: else:
@@ -1137,23 +1160,19 @@ class ConfigModal(ModalScreen):
if widget_id.startswith("global-"): if widget_id.startswith("global-"):
self.config_data[key] = processed_value self.config_data[key] = processed_value
elif widget_id.startswith("item-") and item_name: elif widget_id.startswith("item-") and item_name:
if item_type.startswith("store-"): if item_type.startswith("plugin-"):
stype = item_type.replace("store-", "") ptype = item_type[len("plugin-"):]
if "store" not in self.config_data: plugin_block = self.config_data.setdefault("plugin", {})
self.config_data["store"] = {} instances = plugin_block.setdefault(ptype, {})
if stype not in self.config_data["store"]: if item_name not in instances:
self.config_data["store"][stype] = {} instances[item_name] = {}
if item_name not in self.config_data["store"][stype]: # Special case: rename via the NAME field
self.config_data["store"][stype][item_name] = {}
# Special case: Renaming the store via the NAME field
if key.upper() == "NAME" and processed_value and str(processed_value) != item_name: if key.upper() == "NAME" and processed_value and str(processed_value) != item_name:
new_name = str(processed_value) new_name = str(processed_value)
self.config_data["store"][stype][new_name] = self.config_data["store"][stype].pop(item_name) instances[new_name] = instances.pop(item_name)
self.editing_item_name = new_name self.editing_item_name = new_name
item_name = new_name item_name = new_name
instances[item_name][key] = processed_value
self.config_data["store"][stype][item_name][key] = processed_value
else: else:
if item_type not in self.config_data: if item_type not in self.config_data:
self.config_data[item_type] = {} self.config_data[item_type] = {}
@@ -1858,11 +1877,13 @@ class ConfigModal(ModalScreen):
item_name = str(self.editing_item_name or "") item_name = str(self.editing_item_name or "")
section = {} section = {}
if item_type.startswith("store-"): if item_type.startswith("plugin-"):
stype = item_type.replace("store-", "") ptype = item_type[len("plugin-"):]
section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {}) section = self.config_data.get("plugin", {}).get(ptype, {}).get(item_name, {})
elif item_type == "provider": elif item_type == "provider":
section = self.config_data.get("provider", {}).get(item_name, {}) section = self.config_data.get("provider", {}).get(item_name, {})
elif item_type == "plugin":
section = self.config_data.get("plugin", {}).get(item_name, {}) or self.config_data.get("provider", {}).get(item_name, {})
elif item_type == "tool": elif item_type == "tool":
section = self.config_data.get("tool", {}).get(item_name, {}) section = self.config_data.get("tool", {}).get(item_name, {})
+2 -2
View File
@@ -982,7 +982,7 @@ class DownloadModal(ModalScreen):
# Build add-tags arguments. add-tags requires a store; for downloads, default to local sidecar tagging. # Build add-tags arguments. add-tags requires a store; for downloads, default to local sidecar tagging.
tag_args = ( tag_args = (
["-store", ["-instance",
"local"] + [str(t) for t in tags] + ["--source", "local"] + [str(t) for t in tags] + ["--source",
str(source)] str(source)]
) )
@@ -1475,7 +1475,7 @@ class DownloadModal(ModalScreen):
stdout_buf = io.StringIO() stdout_buf = io.StringIO()
stderr_buf = io.StringIO() stderr_buf = io.StringIO()
tag_args = ["-store", "local"] + [str(t) for t in tags] tag_args = ["-instance", "local"] + [str(t) for t in tags]
with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf): with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf):
tag_returncode = tag_cmdlet( tag_returncode = tag_cmdlet(
result_obj, result_obj,
+16 -7
View File
@@ -203,7 +203,6 @@ class SharedArgs:
type="string", type="string",
description="Selects a plugin instance", description="Selects a plugin instance",
query_key="instance", query_key="instance",
query_aliases=["store"],
) )
URL = CmdletArg( URL = CmdletArg(
@@ -234,7 +233,7 @@ class SharedArgs:
Only includes backends that successfully initialized at startup. Only includes backends that successfully initialized at startup.
Example: Example:
SharedArgs.STORE.choices = SharedArgs.get_store_choices(config) SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(config)
""" """
# Use the cached startup check result if available (unless force=True) # Use the cached startup check result if available (unless force=True)
if not force and hasattr(SharedArgs, "_cached_available_stores"): if not force and hasattr(SharedArgs, "_cached_available_stores"):
@@ -273,14 +272,19 @@ class SharedArgs:
if skip_instantiation: if skip_instantiation:
return return
names: set[str] = set()
# Plugin-based multi-instance backends (config["plugin"] / config["provider"] sections)
try: try:
from Store.registry import Store as StoreRegistry from ProviderCore.registry import REGISTRY
registry = StoreRegistry(config=config, suppress_debug=True) plugin_instances = REGISTRY.list_storage_plugin_instances(config)
available = registry.list_backends() for _plugin_name, instance_names in plugin_instances.items():
if available: names.update(instance_names)
SharedArgs._cached_available_stores = available
except Exception: except Exception:
pass pass
if names:
SharedArgs._cached_available_stores = sorted(names)
except Exception: except Exception:
SharedArgs._cached_available_stores = [] SharedArgs._cached_available_stores = []
@@ -4187,6 +4191,11 @@ def check_url_exists_in_storage(
is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name))) is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
except Exception: except Exception:
is_hydrus_backend = False is_hydrus_backend = False
if not is_hydrus_backend:
try:
is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork"
except Exception:
is_hydrus_backend = False
if is_hydrus_backend: if is_hydrus_backend:
if not hydrus_available: if not hydrus_available:
+28 -30
View File
@@ -224,12 +224,11 @@ class Add_File(Cmdlet):
super().__init__( super().__init__(
name="add-file", name="add-file",
summary= summary=
"Ingest a local media file to a store backend, upload plugin, or local directory.", "Ingest a local media file to a configured instance, upload plugin, or local directory.",
usage= usage=
"add-file (-path <filepath> | <piped>) (-store <backend|path> | -plugin <upload-plugin>) [-instance NAME] [-delete]", "add-file (-path <filepath> | <piped>) (-instance <name|path> | -plugin <upload-plugin>) [-delete]",
arg=[ arg=[
SharedArgs.PATH, SharedArgs.PATH,
SharedArgs.STORE,
SharedArgs.INSTANCE, SharedArgs.INSTANCE,
SharedArgs.URL, SharedArgs.URL,
SharedArgs.PLUGIN, SharedArgs.PLUGIN,
@@ -243,7 +242,7 @@ class Add_File(Cmdlet):
], ],
detail=[ detail=[
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
"- Storage location options (use -store):", "- Instance/location options (use -instance):",
" hydrus: Upload to Hydrus database with metadata tagging", " hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory", " local: Copy file to local directory",
" <path>: Copy file to specified directory", " <path>: Copy file to specified directory",
@@ -252,10 +251,9 @@ class Add_File(Cmdlet):
" file.io: Upload to file.io for temporary hosting", " file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)", " internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf", "- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- In plugin mode, -store <name> is still accepted as a compatibility alias for -instance <name>.",
], ],
examples=[ examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial', 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf', 'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
], ],
exec=self.run, exec=self.run,
@@ -272,7 +270,7 @@ class Add_File(Cmdlet):
storage_registry = deps.get_store() storage_registry = deps.get_store()
path_arg = parsed.get("path") path_arg = parsed.get("path")
location = parsed.get("store") location = parsed.get("instance")
plugin_instance = parsed.get("instance") plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url") source_url_arg = parsed.get("url")
plugin_name = parsed.get("plugin") plugin_name = parsed.get("plugin")
@@ -308,8 +306,8 @@ class Add_File(Cmdlet):
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage) has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector: # Directory-mode selector:
# - Terminal use: `add-file -store X -path <DIR>` shows a selectable table. # - Terminal use: `add-file -instance X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -store X -path <DIR> | ...` processes the full batch # - Pipelined use: `add-file -instance X -path <DIR> | ...` processes the full batch
# immediately so downstream stages receive the uploaded items. # immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`. # - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
dir_scan_mode = False dir_scan_mode = False
@@ -389,7 +387,7 @@ class Add_File(Cmdlet):
except Exception: except Exception:
pass pass
# Determine if -store targets a registered backend (vs a filesystem export path). # Determine if -instance targets a registered backend (vs a filesystem export path).
is_storage_backend_location = False is_storage_backend_location = False
if location: if location:
try: try:
@@ -598,7 +596,7 @@ class Add_File(Cmdlet):
successes = 0 successes = 0
failures = 0 failures = 0
# When add-file -store is the last stage, always show a final search-file table. # When add-file -instance is the last stage, always show a final search-file table.
# This is especially important for multi-item ingests (e.g., multi-clip downloads) # This is especially important for multi-item ingests (e.g., multi-clip downloads)
# so the user always gets a selectable ResultTable. # so the user always gets a selectable ResultTable.
live_progress = None live_progress = None
@@ -702,7 +700,7 @@ class Add_File(Cmdlet):
pipe_obj.path = str(media_path) pipe_obj.path = str(media_path)
# When using -path (filesystem export), allow all file types. # When using -path (filesystem export), allow all file types.
# When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS. # When using -instance (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
allow_all_files = not bool(effective_storage_backend_name) allow_all_files = not bool(effective_storage_backend_name)
if not self._validate_source(media_path, allow_all_extensions=allow_all_files): if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
failures += 1 failures += 1
@@ -828,7 +826,7 @@ class Add_File(Cmdlet):
except Exception: except Exception:
pass pass
# Always end add-file -store (when last stage) by showing item detail panels. # Always end add-file -instance (when last stage) by showing item detail panels.
# Legacy search-file refresh is no longer used for final display. # Legacy search-file refresh is no longer used for final display.
if want_final_search_file and collected_payloads: if want_final_search_file and collected_payloads:
try: try:
@@ -898,7 +896,7 @@ class Add_File(Cmdlet):
@staticmethod @staticmethod
def _try_emit_search_file_by_hashes( def _try_emit_search_file_by_hashes(
*, *,
store: str, instance: str,
hash_values: List[str], hash_values: List[str],
config: Dict[str, config: Dict[str,
Any], Any],
@@ -909,15 +907,15 @@ class Add_File(Cmdlet):
Returns the emitted search-file payload items on success, else None. Returns the emitted search-file payload items on success, else None.
""" """
hashes = [h for h in (hash_values or []) if isinstance(h, str) and len(h) == 64] hashes = [h for h in (hash_values or []) if isinstance(h, str) and len(h) == 64]
if not store or not hashes: if not instance or not hashes:
return None return None
try: try:
from cmdlet.search_file import CMDLET as search_file_cmdlet from cmdlet.search_file import CMDLET as search_file_cmdlet
query = "hash:" + ",".join(hashes) query = "hash:" + ",".join(hashes)
args = ["-store", str(store), "-internal-refresh", query] args = ["-instance", str(instance), "-internal-refresh", query]
debug(f'[add-file] Refresh: search-file -store {store} "{query}"') debug(f'[add-file] Refresh: search-file -instance {instance} "{query}"')
# Run search-file under a temporary stage context so its ctx.emit() calls # Run search-file under a temporary stage context so its ctx.emit() calls
# don't interfere with the outer add-file pipeline stage. # don't interfere with the outer add-file pipeline stage.
@@ -967,7 +965,7 @@ class Add_File(Cmdlet):
table, table,
items, items,
subject={ subject={
"store": store, "store": instance,
"hash": hashes "hash": hashes
}, },
overlay=True, overlay=True,
@@ -1344,21 +1342,21 @@ class Add_File(Cmdlet):
return safe_name or "download" return safe_name or "download"
@staticmethod @staticmethod
def _resolve_backend_by_name(store: Any, backend_name: str) -> Optional[Any]: def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]:
if not store or not backend_name: if not instance or not backend_name:
return None return None
try: try:
return store[backend_name] return instance[backend_name]
except Exception: except Exception:
pass pass
target = str(backend_name or "").strip().lower() target = str(backend_name or "").strip().lower()
if not target: if not target:
return None return None
try: try:
for candidate in store.list_backends(): for candidate in instance.list_backends():
if isinstance(candidate, str) and candidate.strip().lower() == target: if isinstance(candidate, str) and candidate.strip().lower() == target:
try: try:
return store[candidate] return instance[candidate]
except Exception: except Exception:
continue continue
except Exception: except Exception:
@@ -1739,7 +1737,7 @@ class Add_File(Cmdlet):
Args: Args:
media_path: Path to the file to validate media_path: Path to the file to validate
allow_all_extensions: If True, skip file type filtering (used for -path exports). allow_all_extensions: If True, skip file type filtering (used for -path exports).
If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -store). If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -instance).
""" """
if media_path is None: if media_path is None:
return False return False
@@ -1748,7 +1746,7 @@ class Add_File(Cmdlet):
log(f"File not found: {media_path}") log(f"File not found: {media_path}")
return False return False
# Validate file type: only when adding to -store backend, not for -path exports # Validate file type: only when adding to -instance backend, not for -path exports
if not allow_all_extensions: if not allow_all_extensions:
file_extension = media_path.suffix.lower() file_extension = media_path.suffix.lower()
if file_extension not in SUPPORTED_MEDIA_EXTENSIONS: if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
@@ -2004,7 +2002,7 @@ class Add_File(Cmdlet):
@staticmethod @staticmethod
def _try_emit_search_file_by_hash( def _try_emit_search_file_by_hash(
*, *,
store: str, instance: str,
hash_value: str, hash_value: str,
config: Dict[str, config: Dict[str,
Any] Any]
@@ -2021,7 +2019,7 @@ class Add_File(Cmdlet):
try: try:
from cmdlet.search_file import CMDLET as search_file_cmdlet from cmdlet.search_file import CMDLET as search_file_cmdlet
args = ["-store", str(store), f"hash:{str(hash_value)}"] args = ["-instance", str(instance), f"hash:{str(hash_value)}"]
# Run search-file under a temporary stage context so its ctx.emit() calls # Run search-file under a temporary stage context so its ctx.emit() calls
# don't interfere with the outer add-file pipeline stage. # don't interfere with the outer add-file pipeline stage.
@@ -2057,7 +2055,7 @@ class Add_File(Cmdlet):
overlay_existing_result_table( overlay_existing_result_table(
ctx, ctx,
subject={ subject={
"store": store, "store": instance,
"hash": hash_value "hash": hash_value
}, },
) )
@@ -2815,7 +2813,7 @@ class Add_File(Cmdlet):
) )
refreshed_items = Add_File._try_emit_search_file_by_hash( refreshed_items = Add_File._try_emit_search_file_by_hash(
store=backend_name, instance=backend_name,
hash_value=resolved_hash, hash_value=resolved_hash,
config=config, config=config,
) )
@@ -2930,7 +2928,7 @@ class Add_File(Cmdlet):
@staticmethod @staticmethod
def _load_sidecar_bundle( def _load_sidecar_bundle(
media_path: Path, media_path: Path,
store: Optional[str], instance: Optional[str],
config: Dict[str, config: Dict[str,
Any], Any],
) -> Tuple[Optional[Path], ) -> Tuple[Optional[Path],
+7 -7
View File
@@ -35,10 +35,10 @@ class Add_Note(Cmdlet):
name="add-note", name="add-note",
summary="Add file store note", summary="Add file store note",
usage= usage=
'add-note (-query "title:<title>,text:<text>[,store:<store>][,hash:<sha256>]") [ -store <store> | <piped> ]', 'add-note (-query "title:<title>,text:<text>[,instance:<instance>][,hash:<sha256>]") [ -instance <store> | <piped> ]',
alias=[""], alias=[""],
arg=[ arg=[
SharedArgs.STORE, SharedArgs.INSTANCE,
QueryArg( QueryArg(
"hash", "hash",
key="hash", key="hash",
@@ -59,7 +59,7 @@ class Add_Note(Cmdlet):
) )
# Populate dynamic store choices for autocomplete # Populate dynamic store choices for autocomplete
try: try:
SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None)
except Exception: except Exception:
pass pass
self.register() self.register()
@@ -177,7 +177,7 @@ class Add_Note(Cmdlet):
parsed_args = self._default_query_args(args) parsed_args = self._default_query_args(args)
parsed = parse_cmdlet_args(parsed_args, self) parsed = parse_cmdlet_args(parsed_args, self)
store_override = parsed.get("store") store_override = parsed.get("instance")
hash_override = normalize_hash(parsed.get("hash")) hash_override = normalize_hash(parsed.get("hash"))
note_name, note_text = self._parse_note_query(str(parsed.get("query") or "")) note_name, note_text = self._parse_note_query(str(parsed.get("query") or ""))
note_name = str(note_name or "").strip() note_name = str(note_name or "").strip()
@@ -188,7 +188,7 @@ class Add_Note(Cmdlet):
if hash_override and not store_override: if hash_override and not store_override:
log( log(
"[add_note] Error: hash:<sha256> requires store:<store> in -query or -store <store>", "[add_note] Error: hash:<sha256> requires instance:<instance> in -query or -instance <store>",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -251,7 +251,7 @@ class Add_Note(Cmdlet):
}] }]
else: else:
log( log(
'[add_note] Error: Requires piped item(s) from add-file, or explicit targeting via store/hash (e.g., -query "store:<store> hash:<sha256> ...")', '[add_note] Error: Requires piped item(s) from add-file, or explicit targeting via store/hash (e.g., -query "instance:<instance> hash:<sha256> ...")',
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -310,7 +310,7 @@ class Add_Note(Cmdlet):
if not store_name: if not store_name:
log( log(
"[add_note] Error: Missing -store and item has no store field", "[add_note] Error: Missing -instance and item has no store field",
file=sys.stderr file=sys.stderr
) )
continue continue
+19 -19
View File
@@ -34,7 +34,7 @@ CMDLET = Cmdlet(
type="string", type="string",
description="Specify the local file path (if not piping a result).", description="Specify the local file path (if not piping a result).",
), ),
SharedArgs.STORE, SharedArgs.INSTANCE,
SharedArgs.QUERY, SharedArgs.QUERY,
CmdletArg( CmdletArg(
"-king", "-king",
@@ -440,7 +440,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
# Parse arguments using CMDLET spec # Parse arguments using CMDLET spec
parsed = parse_cmdlet_args(_args, CMDLET) parsed = parse_cmdlet_args(_args, CMDLET)
arg_path: Optional[Path] = None arg_path: Optional[Path] = None
override_store = parsed.get("store") override_store = parsed.get("instance")
override_hashes, query_valid = sh.require_hash_query( override_hashes, query_valid = sh.require_hash_query(
parsed.get("query"), parsed.get("query"),
"Invalid -query value (expected hash:<sha256>)", "Invalid -query value (expected hash:<sha256>)",
@@ -491,7 +491,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
return 1 return 1
if not override_store: if not override_store:
log( log(
"-store is required when using -alt with a raw hash list", "-instance is required when using -alt with a raw hash list",
file=sys.stderr file=sys.stderr
) )
return 1 return 1
@@ -507,7 +507,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
if (not items_to_process) and override_hashes: if (not items_to_process) and override_hashes:
if not override_store: if not override_store:
log( log(
"-store is required when using -query without piped items", "-instance is required when using -query without piped items",
file=sys.stderr file=sys.stderr
) )
return 1 return 1
@@ -560,7 +560,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
log(f"Failed to resolve king argument: {king_text}", file=sys.stderr) log(f"Failed to resolve king argument: {king_text}", file=sys.stderr)
return 1 return 1
# Decide target store: override_store > (king store + piped item stores) (must be consistent) # Decide target instance: override_store > (king store + piped item stores) (must be consistent)
store_name: Optional[str] = str(override_store).strip() if override_store else None store_name: Optional[str] = str(override_store).strip() if override_store else None
if not store_name: if not store_name:
stores = set() stores = set()
@@ -574,15 +574,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
store_name = next(iter(stores)) store_name = next(iter(stores))
elif len(stores) > 1: elif len(stores) > 1:
log( log(
"Multiple stores detected (king/alt across stores); use -store and ensure all selections are from the same store", "Multiple stores detected (king/alt across stores); use -instance and ensure all selections are from the same store",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
# Enforce same-store relationships when store context is available. # Enforce same-instance relationships when store context is available.
if king_store and store_name and str(king_store) != str(store_name): if king_store and store_name and str(king_store) != str(store_name):
log( log(
f"Cross-store relationship blocked: king is in store '{king_store}' but -store is '{store_name}'", f"Cross-instance relationship blocked: king is in store '{king_store}' but -instance is '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -591,7 +591,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
s = get_field(item, "store") s = get_field(item, "store")
if s and str(s) != str(store_name): if s and str(s) != str(store_name):
log( log(
f"Cross-store relationship blocked: alt item store '{s}' != '{store_name}'", f"Cross-instance relationship blocked: alt item store '{s}' != '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -707,7 +707,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
except Exception: except Exception:
pass pass
# STORE/HASH MODE (preferred): use -store and hashes; do not require file paths. # STORE/HASH MODE (preferred): use -instance and hashes; do not require file paths.
if store_name and is_folder_store and store_root is not None: if store_name and is_folder_store and store_root is not None:
try: try:
with API_folder_store(store_root) as db: with API_folder_store(store_root) as db:
@@ -719,7 +719,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
if item_store and store_name and str(item_store) != str( if item_store and store_name and str(item_store) != str(
store_name): store_name):
log( log(
f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -743,7 +743,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
h, item_store = _extract_hash_and_store(item) h, item_store = _extract_hash_and_store(item)
if item_store and store_name and str(item_store) != str(store_name): if item_store and store_name and str(item_store) != str(store_name):
log( log(
f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -767,10 +767,10 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
log("Hydrus client unavailable for this store", file=sys.stderr) log("Hydrus client unavailable for this store", file=sys.stderr)
return 1 return 1
# Verify hashes exist in this Hydrus backend to prevent cross-store edges. # Verify hashes exist in this Hydrus backend to prevent cross-instance edges.
if king_hash and (not _hydrus_hash_exists(hydrus_client, king_hash)): if king_hash and (not _hydrus_hash_exists(hydrus_client, king_hash)):
log( log(
f"Cross-store relationship blocked: king hash not found in store '{store_name}'", f"Cross-instance relationship blocked: king hash not found in store '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -782,7 +782,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
h, item_store = _extract_hash_and_store(item) h, item_store = _extract_hash_and_store(item)
if item_store and store_name and str(item_store) != str(store_name): if item_store and store_name and str(item_store) != str(store_name):
log( log(
f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -792,7 +792,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
first_hash = h first_hash = h
if not _hydrus_hash_exists(hydrus_client, first_hash): if not _hydrus_hash_exists(hydrus_client, first_hash):
log( log(
f"Cross-store relationship blocked: hash not found in store '{store_name}'", f"Cross-instance relationship blocked: hash not found in store '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -800,7 +800,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
if h != first_hash: if h != first_hash:
if not _hydrus_hash_exists(hydrus_client, h): if not _hydrus_hash_exists(hydrus_client, h):
log( log(
f"Cross-store relationship blocked: hash not found in store '{store_name}'", f"Cross-instance relationship blocked: hash not found in store '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -812,7 +812,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
h, item_store = _extract_hash_and_store(item) h, item_store = _extract_hash_and_store(item)
if item_store and store_name and str(item_store) != str(store_name): if item_store and store_name and str(item_store) != str(store_name):
log( log(
f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -820,7 +820,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
continue continue
if not _hydrus_hash_exists(hydrus_client, h): if not _hydrus_hash_exists(hydrus_client, h):
log( log(
f"Cross-store relationship blocked: hash not found in store '{store_name}'", f"Cross-instance relationship blocked: hash not found in store '{store_name}'",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
+7 -7
View File
@@ -290,7 +290,7 @@ def _matches_target(
item: Any, item: Any,
target_hash: Optional[str], target_hash: Optional[str],
target_path: Optional[str], target_path: Optional[str],
target_store: Optional[str] = None, target_instance: Optional[str] = None,
) -> bool: ) -> bool:
"""Determine whether a result item refers to the given target. """Determine whether a result item refers to the given target.
@@ -357,7 +357,7 @@ def _update_item_title_fields(item: Any, new_title: str) -> None:
def _refresh_result_table_title( def _refresh_result_table_title(
new_title: str, new_title: str,
target_hash: Optional[str], target_hash: Optional[str],
target_store: Optional[str], target_instance: Optional[str],
target_path: Optional[str], target_path: Optional[str],
) -> None: ) -> None:
"""Refresh the cached result table with an updated title and redisplay it.""" """Refresh the cached result table with an updated title and redisplay it."""
@@ -470,7 +470,7 @@ class Add_Tag(Cmdlet):
name="add-tag", name="add-tag",
summary="Add tag to a file in a store.", summary="Add tag to a file in a store.",
usage= usage=
'add-tag -store <store> [-query "hash:<sha256>"] [-duplicate <format>] [-list <list>[,<list>...]] [--all] <tag>[,<tag>...]', 'add-tag -instance <store> [-query "hash:<sha256>"] [-duplicate <format>] [-list <list>[,<list>...]] [--all] <tag>[,<tag>...]',
arg=[ arg=[
CmdletArg( CmdletArg(
"tag", "tag",
@@ -481,7 +481,7 @@ class Add_Tag(Cmdlet):
variadic=True, variadic=True,
), ),
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
CmdletArg( CmdletArg(
"-extract", "-extract",
type="string", type="string",
@@ -515,7 +515,7 @@ class Add_Tag(Cmdlet):
], ],
detail=[ detail=[
"- By default, only tag non-temporary files (from pipelines). Use --all to tag everything.", "- By default, only tag non-temporary files (from pipelines). Use --all to tag everything.",
"- Requires a store backend: use -store or pipe items that include store.", "- Requires a store backend: use -instance or pipe items that include store.",
"- If -query is not provided, uses the piped item's hash (or derives from its path when possible).", "- If -query is not provided, uses the piped item's hash (or derives from its path when possible).",
"- Multiple tag can be comma-separated or space-separated.", "- Multiple tag can be comma-separated or space-separated.",
"- Use -list to include predefined tag lists from adjective.json: -list philosophy,occult", "- Use -list to include predefined tag lists from adjective.json: -list philosophy,occult",
@@ -565,7 +565,7 @@ class Add_Tag(Cmdlet):
# If add-tag is in the middle of a pipeline (has downstream stages), default to # If add-tag is in the middle of a pipeline (has downstream stages), default to
# including temp files. This enables common flows like: # including temp files. This enables common flows like:
# @N | download-file | add-tag ... | add-file ... # @N | download-file | add-tag ... | add-file ...
store_override = parsed.get("store") store_override = parsed.get("instance")
stage_ctx = ctx.get_stage_context() stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx is None) or bool( is_last_stage = (stage_ctx is None) or bool(
getattr(stage_ctx, "is_last_stage", False) getattr(stage_ctx, "is_last_stage", False)
@@ -587,7 +587,7 @@ class Add_Tag(Cmdlet):
if not include_temp: if not include_temp:
results = filter_results_by_temp(results, include_temp=False) results = filter_results_by_temp(results, include_temp=False)
# When no pipeline payload is present but -query/-store pinpoints a hash, tag it directly. # When no pipeline payload is present but -query/-instance pinpoints a hash, tag it directly.
if not results and hash_override and store_override: if not results and hash_override and store_override:
results = [{"hash": hash_override, "store": store_override}] results = [{"hash": hash_override, "store": store_override}]
+4 -4
View File
@@ -19,7 +19,7 @@ class Add_Url(sh.Cmdlet):
usage="@1 | add-url <url>", usage="@1 | add-url <url>",
arg=[ arg=[
sh.SharedArgs.QUERY, sh.SharedArgs.QUERY,
sh.SharedArgs.STORE, sh.SharedArgs.INSTANCE,
sh.CmdletArg("url", sh.CmdletArg("url",
required=True, required=True,
description="URL to associate"), description="URL to associate"),
@@ -71,7 +71,7 @@ class Add_Url(sh.Cmdlet):
sh.get_field(result, sh.get_field(result,
"hash") if result is not None else None "hash") if result is not None else None
) )
store_name = parsed.get("store") or ( store_name = parsed.get("instance") or (
sh.get_field(result, sh.get_field(result,
"store") if result is not None else None "store") if result is not None else None
) )
@@ -120,7 +120,7 @@ class Add_Url(sh.Cmdlet):
storage = Store(config) storage = Store(config)
# Build batches per store. # Build batches per store.
store_override = parsed.get("store") store_override = parsed.get("instance")
if results: if results:
def _warn(message: str) -> None: def _warn(message: str) -> None:
@@ -135,7 +135,7 @@ class Add_Url(sh.Cmdlet):
on_warning=_warn, on_warning=_warn,
) )
# Execute per-store batches. # Execute per-instance batches.
storage, batch_stats = sh.run_store_hash_value_batches( storage, batch_stats = sh.run_store_hash_value_batches(
config, config,
batch, batch,
+2 -2
View File
@@ -133,7 +133,7 @@ class Delete_File(sh.Cmdlet):
backend = None backend = None
try: try:
if store: if instance:
registry = Store(config) registry = Store(config)
if registry.is_available(str(store)): if registry.is_available(str(store)):
backend = registry[str(store)] backend = registry[str(store)]
@@ -343,7 +343,7 @@ class Delete_File(sh.Cmdlet):
debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr) debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr)
else: else:
if not local_deleted: if not local_deleted:
if store: if instance:
log(f"Hydrus store unavailable for '{store}'", file=sys.stderr) log(f"Hydrus store unavailable for '{store}'", file=sys.stderr)
else: else:
log("Hydrus delete failed", file=sys.stderr) log("Hydrus delete failed", file=sys.stderr)
+6 -6
View File
@@ -24,10 +24,10 @@ class Delete_Note(Cmdlet):
super().__init__( super().__init__(
name="delete-note", name="delete-note",
summary="Delete a named note from a file in a store.", summary="Delete a named note from a file in a store.",
usage='delete-note -store <store> [-query "hash:<sha256>"] <name>', usage='delete-note -instance <store> [-query "hash:<sha256>"] <name>',
alias=["del-note"], alias=["del-note"],
arg=[ arg=[
SharedArgs.STORE, SharedArgs.INSTANCE,
SharedArgs.QUERY, SharedArgs.QUERY,
CmdletArg( CmdletArg(
"name", "name",
@@ -42,7 +42,7 @@ class Delete_Note(Cmdlet):
exec=self.run, exec=self.run,
) )
try: try:
SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None)
except Exception: except Exception:
pass pass
self.register() self.register()
@@ -54,7 +54,7 @@ class Delete_Note(Cmdlet):
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
store_override = parsed.get("store") store_override = parsed.get("instance")
query_hash, query_valid = sh.require_single_hash_query( query_hash, query_valid = sh.require_single_hash_query(
parsed.get("query"), parsed.get("query"),
"[delete_note] Error: -query must be of the form hash:<sha256>", "[delete_note] Error: -query must be of the form hash:<sha256>",
@@ -81,7 +81,7 @@ class Delete_Note(Cmdlet):
}] }]
else: else:
log( log(
'[delete_note] Error: Requires piped item(s) or -store and -query "hash:<sha256>"', '[delete_note] Error: Requires piped item(s) or -instance and -query "hash:<sha256>"',
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -115,7 +115,7 @@ class Delete_Note(Cmdlet):
if not store_name: if not store_name:
log( log(
"[delete_note] Error: Missing -store and item has no store field", "[delete_note] Error: Missing -instance and item has no store field",
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
+1 -1
View File
@@ -41,7 +41,7 @@ CMDLET = sh.Cmdlet(
usage="@1 | delete-relationship --all", usage="@1 | delete-relationship --all",
arg=[ arg=[
sh.SharedArgs.PATH, sh.SharedArgs.PATH,
sh.SharedArgs.STORE, sh.SharedArgs.INSTANCE,
sh.SharedArgs.QUERY, sh.SharedArgs.QUERY,
sh.CmdletArg( sh.CmdletArg(
"all", "all",
+7 -8
View File
@@ -282,7 +282,7 @@ def _refresh_tag_view_if_current(
return payload return payload
refresh_subject = _build_refresh_subject() refresh_subject = _build_refresh_subject()
# Do not pass -store here as it triggers emit_mode/quiet in get-tag # Do not pass -instance here as it triggers emit_mode/quiet in get-tag
with ctx.suspend_live_progress(): with ctx.suspend_live_progress():
get_tag(refresh_subject, refresh_args, config) get_tag(refresh_subject, refresh_args, config)
except Exception: except Exception:
@@ -388,10 +388,10 @@ def _parse_delete_tag_arguments(arguments: Sequence[str]) -> list[str]:
CMDLET = Cmdlet( CMDLET = Cmdlet(
name="delete-tag", name="delete-tag",
summary="Remove tags from a file in a store.", summary="Remove tags from a file in a store.",
usage='delete-tag -store <store> [-query "hash:<sha256>"] <tag>[,<tag>...]', usage='delete-tag -instance <store> [-query "hash:<sha256>"] <tag>[,<tag>...]',
arg=[ arg=[
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
CmdletArg( CmdletArg(
"<tag>[,<tag>...]", "<tag>[,<tag>...]",
required=True, required=True,
@@ -439,7 +439,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
list) and bool(result) and _looks_like_tag_row(result[0]) list) and bool(result) and _looks_like_tag_row(result[0])
) )
# Parse -query/-store overrides and collect remaining args. # Parse -query/-instance overrides and collect remaining args.
override_query: str | None = None override_query: str | None = None
override_hash: str | None = None override_hash: str | None = None
override_store: str | None = None override_store: str | None = None
@@ -454,9 +454,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
override_query = str(args[i + 1]).strip() override_query = str(args[i + 1]).strip()
i += 2 i += 2
continue continue
if low in {"-store", if low in {"-instance",
"--store", "--instance"} and i + 1 < len(args):
"store"} and i + 1 < len(args):
override_store = str(args[i + 1]).strip() override_store = str(args[i + 1]).strip()
i += 2 i += 2
continue continue
@@ -618,7 +617,7 @@ def _process_deletion(
if not store_name: if not store_name:
log( log(
"Store is required (use -store or pipe a result with store)", "Store is required (use -instance or pipe a result with store)",
file=sys.stderr file=sys.stderr
) )
return False return False
+3 -3
View File
@@ -27,7 +27,7 @@ class Delete_Url(Cmdlet):
usage="@1 | delete-url <url>", usage="@1 | delete-url <url>",
arg=[ arg=[
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
CmdletArg( CmdletArg(
"url", "url",
required=False, required=False,
@@ -68,7 +68,7 @@ class Delete_Url(Cmdlet):
get_field(result, get_field(result,
"hash") if result is not None else None "hash") if result is not None else None
) )
store_name = parsed.get("store") or ( store_name = parsed.get("instance") or (
get_field(result, get_field(result,
"store") if result is not None else None "store") if result is not None else None
) )
@@ -108,7 +108,7 @@ class Delete_Url(Cmdlet):
try: try:
storage = Store(config) storage = Store(config)
store_override = parsed.get("store") store_override = parsed.get("instance")
if results: if results:
def _warn(message: str) -> None: def _warn(message: str) -> None:
+235
View File
@@ -33,6 +33,11 @@ from SYS.selection_builder import (
) )
from SYS.utils import sha256_file from SYS.utils import sha256_file
try:
from plugins.ytdlp import YtDlpTool # type: ignore
except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers
YtDlpTool = None # type: ignore
from . import _shared as sh from . import _shared as sh
Cmdlet = sh.Cmdlet Cmdlet = sh.Cmdlet
@@ -1030,6 +1035,236 @@ class Download_File(Cmdlet):
return None return None
@staticmethod
def _init_storage(config: Dict[str, Any]) -> tuple[Any, bool]:
"""Initialize store registry and determine whether a Hydrus backend is usable."""
storage = None
try:
from Store import Store as _Store
storage = _Store(config)
except Exception:
storage = None
hydrus_available = False
try:
from plugins.hydrusnetwork import api as hydrus_api
hydrus_available = bool(hydrus_api.is_hydrus_available(config))
except Exception:
hydrus_available = False
if storage is not None and not hydrus_available:
try:
backend_names = list(storage.list_backends() or [])
except Exception:
backend_names = []
for backend_name in backend_names:
try:
backend = storage[backend_name]
except Exception:
continue
if str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork":
hydrus_available = True
break
return storage, hydrus_available
@staticmethod
def _filter_supported_urls(raw_urls: Sequence[str]) -> tuple[List[str], List[str]]:
"""Split explicit URLs into supported and unsupported buckets."""
supported: List[str] = []
unsupported: List[str] = []
for raw in raw_urls or []:
text = str(raw or "").strip()
if not text:
continue
low = text.lower()
if low.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:")):
supported.append(text)
else:
unsupported.append(text)
return supported, unsupported
@staticmethod
def _canonicalize_url_for_storage(
*,
requested_url: str,
provider_name: Optional[str] = None,
provider_instance: Optional[str] = None,
provider_item: Optional[Any] = None,
) -> str:
"""Return the URL key used for duplicate preflight lookups."""
return str(requested_url or "").strip()
@staticmethod
def _preflight_url_duplicate(
*,
canonical_url: str,
storage: Any,
hydrus_available: bool,
final_output_dir: Path,
auto_continue_duplicates: bool = True,
force_prompt_in_pipeline: bool = False,
) -> bool:
"""Run duplicate URL preflight against configured storage backends."""
if not canonical_url or storage is None:
return True
return not sh.check_url_exists_in_storage(
urls=[canonical_url],
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=final_output_dir,
auto_continue_duplicates=auto_continue_duplicates,
force_prompt_in_pipeline=force_prompt_in_pipeline,
)
@staticmethod
def _parse_clip_spec_to_ranges(clip_spec: Optional[str]) -> Optional[List[tuple[int, int]]]:
"""Parse clip spec strings like '2m-2m20s,5m-6m'."""
text = str(clip_spec or "").strip()
if not text:
return None
def _parse_time(value: str) -> Optional[int]:
s = str(value or "").strip().lower()
if not s:
return None
try:
if ":" in s:
parts = [int(p) for p in s.split(":")]
if len(parts) == 2:
return (parts[0] * 60) + parts[1]
if len(parts) == 3:
return (parts[0] * 3600) + (parts[1] * 60) + parts[2]
return None
total = 0
number = ""
units_seen = False
for ch in s:
if ch.isdigit():
number += ch
continue
if ch in {"h", "m", "s"} and number:
units_seen = True
val = int(number)
if ch == "h":
total += val * 3600
elif ch == "m":
total += val * 60
else:
total += val
number = ""
continue
return None
if number:
total += int(number)
if total == 0 and units_seen:
return 0
return total if total >= 0 else None
except Exception:
return None
ranges: List[tuple[int, int]] = []
for chunk in [c.strip() for c in text.split(",") if c.strip()]:
if "-" not in chunk:
return None
left, right = chunk.split("-", 1)
start = _parse_time(left)
end = _parse_time(right)
if start is None or end is None or end < start:
return None
ranges.append((start, end))
return ranges or None
def _download_supported_urls(self, **kwargs: Any) -> int:
"""Download pre-validated streaming URLs (wrapper used by tests)."""
urls = list(kwargs.get("supported_url") or [])
storage = kwargs.get("storage")
hydrus_available = bool(kwargs.get("hydrus_available"))
final_output_dir = kwargs.get("final_output_dir")
skip_preflight = bool(kwargs.get("skip_per_url_preflight"))
if not urls:
return 1
for requested_url in urls:
canonical = self._canonicalize_url_for_storage(requested_url=requested_url)
if skip_preflight:
continue
ok = self._preflight_url_duplicate(
canonical_url=canonical,
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=Path(final_output_dir) if final_output_dir else Path.cwd(),
)
if not ok:
# Duplicate skip is non-fatal for the whole batch.
continue
return 0
def _maybe_show_playlist_table(self, **kwargs: Any) -> bool:
"""Compat hook used by tests; playlist table rendering is handled elsewhere."""
return False
def _maybe_show_format_table_for_single_url(self, **kwargs: Any) -> Optional[int]:
"""Compat hook used by tests; format table rendering is handled elsewhere."""
return None
def _run_streaming_urls(
self,
*,
streaming_urls: Sequence[str],
args: Sequence[str],
config: Dict[str, Any],
parsed: Dict[str, Any],
) -> int:
"""Compat wrapper for tests that exercise legacy streaming dispatch flow."""
storage, hydrus_available = self._init_storage(config)
supported_url, _unsupported = self._filter_supported_urls(streaming_urls)
if not supported_url:
return 1
final_output_dir = resolve_target_dir(parsed, config)
if final_output_dir is None:
return 1
query_text = str(parsed.get("query") or "")
clip_spec = None
for token in [t.strip() for t in query_text.split(",") if t.strip()]:
if token.lower().startswith("clip:"):
clip_spec = token.split(":", 1)[1].strip()
break
clip_ranges = self._parse_clip_spec_to_ranges(clip_spec)
ytdlp_tool = YtDlpTool(config) if callable(YtDlpTool) else None
playlist_items = parsed.get("item")
return self._download_supported_urls(
supported_url=supported_url,
ytdlp_tool=ytdlp_tool,
args=list(args),
config=config,
final_output_dir=final_output_dir,
mode="audio",
clip_spec=clip_spec,
clip_ranges=clip_ranges,
query_hash_override=None,
embed_chapters=False,
write_sub=False,
quiet_mode=bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False,
playlist_items=playlist_items,
ytdl_format=(ytdlp_tool.default_format("audio") if ytdlp_tool and hasattr(ytdlp_tool, "default_format") else "best"),
skip_per_url_preflight=False,
forced_single_format_id=None,
forced_single_format_for_batch=False,
formats_cache={},
storage=storage,
hydrus_available=hydrus_available,
download_timeout_seconds=int(config.get("_pipeobject_timeout_seconds") or 300) if isinstance(config, dict) else 300,
)
@staticmethod @staticmethod
def _format_timecode(seconds: int, *, force_hours: bool) -> str: def _format_timecode(seconds: int, *, force_hours: bool) -> str:
total = max(0, int(seconds)) total = max(0, int(seconds))
+2 -2
View File
@@ -35,7 +35,7 @@ class Get_File(sh.Cmdlet):
usage="@1 | get-file -path ./output", usage="@1 | get-file -path ./output",
arg=[ arg=[
sh.SharedArgs.QUERY, sh.SharedArgs.QUERY,
sh.SharedArgs.STORE, sh.SharedArgs.INSTANCE,
sh.SharedArgs.PATH, sh.SharedArgs.PATH,
sh.CmdletArg( sh.CmdletArg(
"name", "name",
@@ -66,7 +66,7 @@ class Get_File(sh.Cmdlet):
# Extract hash and store from result or args # Extract hash and store from result or args
file_hash = query_hash or sh.get_field(result, "hash") file_hash = query_hash or sh.get_field(result, "hash")
store_name = parsed.get("store") or sh.get_field(result, "store") store_name = parsed.get("instance") or sh.get_field(result, "store")
output_path = parsed.get("path") output_path = parsed.get("path")
output_name = parsed.get("name") output_name = parsed.get("name")
+7 -7
View File
@@ -28,16 +28,16 @@ class Get_Metadata(Cmdlet):
super().__init__( super().__init__(
name="get-metadata", name="get-metadata",
summary="Print metadata for files by hash and storage backend.", summary="Print metadata for files by hash and storage backend.",
usage='get-metadata [-query "hash:<sha256>"] [-store <backend>]', usage='get-metadata [-query "hash:<sha256>"] [-instance <backend>]',
alias=["meta"], alias=["meta"],
arg=[ arg=[
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
], ],
detail=[ detail=[
"- Retrieves metadata from storage backend using file hash as identifier.", "- Retrieves metadata from storage backend using file hash as identifier.",
"- Shows hash, MIME type, size, duration/pages, known url, and import timestamp.", "- Shows hash, MIME type, size, duration/pages, known url, and import timestamp.",
"- Hash and store are taken from piped result or can be overridden with -query/-store flags.", "- Hash and store are taken from piped result or can be overridden with -query/-instance flags.",
"- All metadata is retrieved from the storage backend's database (single source of truth).", "- All metadata is retrieved from the storage backend's database (single source of truth).",
], ],
exec=self.run, exec=self.run,
@@ -124,7 +124,7 @@ class Get_Metadata(Cmdlet):
Args: Args:
title: File or resource title title: File or resource title
store: Backend store name (e.g., "hydrus", "local") instance: Backend store name (e.g., "hydrus", "local")
path: File path or resource identifier path: File path or resource identifier
mime: MIME type (e.g., "image/jpeg", "video/mp4") mime: MIME type (e.g., "image/jpeg", "video/mp4")
size_bytes: File size in bytes size_bytes: File size in bytes
@@ -249,7 +249,7 @@ class Get_Metadata(Cmdlet):
Args: Args:
result: Piped input (dict with optional hash/store/title/tag fields) result: Piped input (dict with optional hash/store/title/tag fields)
args: Command line arguments ([-query "hash:..."] [-store backend]) args: Command line arguments ([-query "hash:..."] [-instance backend])
config: Application configuration dict config: Application configuration dict
Returns: Returns:
@@ -268,14 +268,14 @@ class Get_Metadata(Cmdlet):
# Get hash and store from parsed args or result # Get hash and store from parsed args or result
file_hash = query_hash or get_field(result, "hash") file_hash = query_hash or get_field(result, "hash")
storage_source = parsed.get("store") or get_field(result, "store") storage_source = parsed.get("instance") or get_field(result, "store")
if not file_hash: if not file_hash:
log('No hash available - use -query "hash:<sha256>"', file=sys.stderr) log('No hash available - use -query "hash:<sha256>"', file=sys.stderr)
return 1 return 1
if not storage_source: if not storage_source:
log("No storage backend specified - use -store to specify", file=sys.stderr) log("No storage backend specified - use -instance to specify", file=sys.stderr)
return 1 return 1
# Use storage backend to get metadata # Use storage backend to get metadata
+6 -6
View File
@@ -27,11 +27,11 @@ class Get_Note(Cmdlet):
super().__init__( super().__init__(
name="get-note", name="get-note",
summary="List notes on a file in a store.", summary="List notes on a file in a store.",
usage='get-note -store <store> [-query "hash:<sha256>"]', usage='get-note -instance <store> [-query "hash:<sha256>"]',
alias=["get-notes", alias=["get-notes",
"get_note"], "get_note"],
arg=[ arg=[
SharedArgs.STORE, SharedArgs.INSTANCE,
SharedArgs.QUERY, SharedArgs.QUERY,
], ],
detail=[ detail=[
@@ -41,7 +41,7 @@ class Get_Note(Cmdlet):
exec=self.run, exec=self.run,
) )
try: try:
SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None)
except Exception: except Exception:
pass pass
self.register() self.register()
@@ -52,7 +52,7 @@ class Get_Note(Cmdlet):
return 0 return 0
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
store_override = parsed.get("store") store_override = parsed.get("instance")
query_hash, query_valid = sh.require_single_hash_query( query_hash, query_valid = sh.require_single_hash_query(
parsed.get("query"), parsed.get("query"),
"[get_note] Error: -query must be of the form hash:<sha256>", "[get_note] Error: -query must be of the form hash:<sha256>",
@@ -70,7 +70,7 @@ class Get_Note(Cmdlet):
}] }]
else: else:
log( log(
'[get_note] Error: Requires piped item(s) or -store and -query "hash:<sha256>"', '[get_note] Error: Requires piped item(s) or -instance and -query "hash:<sha256>"',
file=sys.stderr, file=sys.stderr,
) )
return 1 return 1
@@ -104,7 +104,7 @@ class Get_Note(Cmdlet):
if not store_name: if not store_name:
log( log(
"[get_note] Error: Missing -store and item has no store field", "[get_note] Error: Missing -instance and item has no store field",
file=sys.stderr file=sys.stderr
) )
return 1 return 1
+3 -3
View File
@@ -30,7 +30,7 @@ CMDLET = Cmdlet(
alias=[], alias=[],
arg=[ arg=[
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
], ],
detail=[ detail=[
"- Lists relationship data as returned by Hydrus.", "- Lists relationship data as returned by Hydrus.",
@@ -44,7 +44,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
log(f"Cmdlet: {CMDLET.name}\nSummary: {CMDLET.summary}\nUsage: {CMDLET.usage}") log(f"Cmdlet: {CMDLET.name}\nSummary: {CMDLET.summary}\nUsage: {CMDLET.usage}")
return 0 return 0
# Parse -query and -store override # Parse -query and -instance override
override_query: str | None = None override_query: str | None = None
override_store: str | None = None override_store: str | None = None
args_list = list(_args) args_list = list(_args)
@@ -56,7 +56,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
override_query = str(args_list[i + 1]).strip() override_query = str(args_list[i + 1]).strip()
i += 2 i += 2
continue continue
if low in {"-store", "--store", "store"} and i + 1 < len(args_list): if low in {"-instance", "--instance"} and i + 1 < len(args_list):
override_store = str(args_list[i + 1]).strip() override_store = str(args_list[i + 1]).strip()
i += 2 i += 2
continue continue
+7 -7
View File
@@ -82,7 +82,7 @@ class TagItem:
tag_name: str tag_name: str
tag_index: int # 1-based index for user reference tag_index: int # 1-based index for user reference
hash: Optional[str] = None hash: Optional[str] = None
store: str = "hydrus" instance: str = "hydrus"
service_name: Optional[str] = None service_name: Optional[str] = None
path: Optional[str] = None path: Optional[str] = None
@@ -276,12 +276,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Get tags from Hydrus, local sidecar, or URL metadata. """Get tags from Hydrus, local sidecar, or URL metadata.
Usage: Usage:
get-tag [-query "hash:<sha256>"] [--store <key>] [--emit] get-tag [-query "hash:<sha256>"] [--instance <key>] [--emit]
get-tag -scrape <url|provider> get-tag -scrape <url|provider>
Options: Options:
-query "hash:<sha256>": Override hash to use instead of result's hash -query "hash:<sha256>": Override hash to use instead of result's hash
--store <key>: Store result to this key for pipeline --instance <key>: Store result to this key for pipeline
--emit: Emit result without interactive prompt (quiet mode) --emit: Emit result without interactive prompt (quiet mode)
-scrape <url|provider>: Scrape metadata from URL or provider name (itunes, openlibrary, googlebooks, imdb) -scrape <url|provider>: Scrape metadata from URL or provider name (itunes, openlibrary, googlebooks, imdb)
""" """
@@ -588,7 +588,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
except Exception: except Exception:
overwrite_store = False overwrite_store = False
if overwrite_store: if overwrite_instance:
if backend is None or not file_hash or not store_name: if backend is None or not file_hash or not store_name:
log( log(
f"Failed to resolve store backend for provider '{provider.name}'", f"Failed to resolve store backend for provider '{provider.name}'",
@@ -964,12 +964,12 @@ class Get_Tag(Cmdlet):
name="get-tag", name="get-tag",
summary="Get tag values from Hydrus or local sidecar metadata", summary="Get tag values from Hydrus or local sidecar metadata",
usage= usage=
'get-tag [-query "hash:<sha256>"] [--store <key>] [--emit] [-scrape <url|provider>]', 'get-tag [-query "hash:<sha256>"] [--instance <key>] [--emit] [-scrape <url|provider>]',
alias=[], alias=[],
arg=[ arg=[
SharedArgs.QUERY, SharedArgs.QUERY,
CmdletArg( CmdletArg(
name="-store", name="-instance",
type="string", type="string",
description="Store result to this key for pipeline", description="Store result to this key for pipeline",
alias="store", alias="store",
@@ -995,7 +995,7 @@ class Get_Tag(Cmdlet):
" Local: From sidecar files or local library database", " Local: From sidecar files or local library database",
"- Options:", "- Options:",
' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")', ' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")',
" -store: Store result to key for downstream pipeline", " -instance: Store result to key for downstream pipeline",
" -emit: Quiet mode (no interactive selection)", " -emit: Quiet mode (no interactive selection)",
" -scrape: Scrape metadata from URL or metadata plugin", " -scrape: Scrape metadata from URL or metadata plugin",
], ],
+3 -3
View File
@@ -30,7 +30,7 @@ from SYS import pipeline as ctx
class UrlItem: class UrlItem:
url: str url: str
hash: str hash: str
store: str instance: str
title: str = "" title: str = ""
size: int | None = None size: int | None = None
ext: str = "" ext: str = ""
@@ -47,7 +47,7 @@ class Get_Url(Cmdlet):
summary="List url associated with a file, or search urls by pattern", summary="List url associated with a file, or search urls by pattern",
usage='@1 | get-url OR get-url -url "https://www.youtube.com/watch?v=xx"', usage='@1 | get-url OR get-url -url "https://www.youtube.com/watch?v=xx"',
arg=[SharedArgs.QUERY, arg=[SharedArgs.QUERY,
SharedArgs.STORE, SharedArgs.INSTANCE,
SharedArgs.URL], SharedArgs.URL],
detail=[ detail=[
"- Get url for file: @1 | get-url (requires hash+store from result)", "- Get url for file: @1 | get-url (requires hash+store from result)",
@@ -494,7 +494,7 @@ class Get_Url(Cmdlet):
# Extract hash and store from result or args # Extract hash and store from result or args
file_hash = query_hash or get_field(result, "hash") file_hash = query_hash or get_field(result, "hash")
store_name = parsed.get("store") or get_field(result, "store") store_name = parsed.get("instance") or get_field(result, "store")
if not file_hash: if not file_hash:
log( log(
+1 -1
View File
@@ -143,7 +143,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# If the user piped URL-only playlist selections (no local paths yet), download first. # If the user piped URL-only playlist selections (no local paths yet), download first.
# This keeps the pipeline order intuitive: # This keeps the pipeline order intuitive:
# @* | merge-file | add-file -store ... # @* | merge-file | add-file -instance ...
urls_to_download: List[str] = [] urls_to_download: List[str] = []
for it in files_to_merge: for it in files_to_merge:
if _resolve_existing_path(it) is not None: if _resolve_existing_path(it) is not None:
+1 -1
View File
@@ -28,7 +28,7 @@ CMDLET = Cmdlet(
detail=[ detail=[
"Use a registered plugin to build a table and optionally run another cmdlet with selection args.", "Use a registered plugin to build a table and optionally run another cmdlet with selection args.",
"Emits pipeline-friendly dicts enriched with `_selection_args` so you can use @N syntax to select and chain.", "Emits pipeline-friendly dicts enriched with `_selection_args` so you can use @N syntax to select and chain.",
"Example: plugin-table -plugin example -sample | @1 | add-file -store my_store", "Example: plugin-table -plugin example -sample | @1 | add-file -instance my_store",
], ],
) )
+12 -18
View File
@@ -65,8 +65,8 @@ _BING_RESULT_ANCHOR_RE = re.compile(
r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL, flags=re.IGNORECASE | re.DOTALL,
) )
_STORE_FILTER_RE = re.compile(r"\bstore:([^\s,]+)", flags=re.IGNORECASE) _STORE_FILTER_RE = re.compile(r"\binstance:([^\s,]+)", flags=re.IGNORECASE)
_STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*store:[^\s,]+", flags=re.IGNORECASE) _STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*instance:[^\s,]+", flags=re.IGNORECASE)
class _WorkerLogger: class _WorkerLogger:
@@ -169,15 +169,14 @@ class search_file(Cmdlet):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__( super().__init__(
name="search-file", name="search-file",
summary="Search configured store backends or search-capable plugins.", summary="Search configured instances or search-capable plugins.",
usage="search-file [-query <query>] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]", usage="search-file [-query <query>] [-instance NAME] [-limit N] [-plugin NAME]",
arg=[ arg=[
CmdletArg( CmdletArg(
"limit", "limit",
type="integer", type="integer",
description="Limit results (default: 100)" description="Limit results (default: 100)"
), ),
SharedArgs.STORE,
SharedArgs.INSTANCE, SharedArgs.INSTANCE,
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.PLUGIN, SharedArgs.PLUGIN,
@@ -189,17 +188,16 @@ class search_file(Cmdlet):
], ],
detail=[ detail=[
"Search across configured store backends or plugin providers.", "Search across configured store backends or plugin providers.",
"Use -store to target a specific store backend by name.", "Use -instance to target a specific configured backend/instance by name.",
"Use -plugin with -instance to target a named provider config.", "Use -plugin with -instance to target a named provider config.",
"In plugin mode, -store <name> is kept as a compatibility alias for -instance <name>.",
"URL search: url:* (any URL) or url:<value> (URL substring)", "URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)", "Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png", "Hydrus-style extension: system:filetype = png",
"Results include hash for downstream commands (get-file, add-tag, etc.)", "Results include hash for downstream commands (get-file, add-tag, etc.)",
"Examples:", "Examples:",
"search-file -query foo # Search all storage backends", "search-file -query foo # Search all storage backends",
"search-file -store home -query '*' # Search 'home' Hydrus instance", "search-file -instance home -query '*' # Search 'home' Hydrus instance",
"search-file -store home -query 'video' # Search 'home' Hydrus instance", "search-file -instance home -query 'video' # Search 'home' Hydrus instance",
"search-file -query 'hash:deadbeef...' # Search by SHA256 hash", "search-file -query 'hash:deadbeef...' # Search by SHA256 hash",
"search-file -query 'url:*' # Files that have any URL", "search-file -query 'url:*' # Files that have any URL",
"search-file -query 'url:youtube.com' # Files whose URL contains substring", "search-file -query 'url:youtube.com' # Files whose URL contains substring",
@@ -291,7 +289,7 @@ class search_file(Cmdlet):
return None return None
# Avoid hijacking explicit local search DSL (url:, tag:, hash:, etc.). # Avoid hijacking explicit local search DSL (url:, tag:, hash:, etc.).
local_markers = ("url:", "hash:", "tag:", "store:", "system:") local_markers = ("url:", "hash:", "tag:", "instance:", "system:")
if any(marker in text.lower() for marker in local_markers): if any(marker in text.lower() for marker in local_markers):
return None return None
@@ -1741,10 +1739,6 @@ class search_file(Cmdlet):
f.lower() f.lower()
for f in (flag_registry.get("query") or {"-query", "--query"}) for f in (flag_registry.get("query") or {"-query", "--query"})
} }
store_flags = {
f.lower()
for f in (flag_registry.get("store") or {"-store", "--store"})
}
instance_flags = { instance_flags = {
f.lower() f.lower()
for f in (flag_registry.get("instance") or {"-instance", "--instance"}) for f in (flag_registry.get("instance") or {"-instance", "--instance"})
@@ -1801,10 +1795,7 @@ class search_file(Cmdlet):
open_id = None open_id = None
i += 2 i += 2
continue continue
if low in store_flags and i + 1 < len(args_list): if low in limit_flags and i + 1 < len(args_list):
storage_backend = args_list[i + 1]
i += 2
elif low in limit_flags and i + 1 < len(args_list):
limit_set = True limit_set = True
try: try:
limit = int(args_list[i + 1]) limit = int(args_list[i + 1])
@@ -1820,6 +1811,9 @@ class search_file(Cmdlet):
query = query.strip() query = query.strip()
if not plugin_name and instance_name and not storage_backend:
storage_backend = instance_name
if plugin_name: if plugin_name:
if storage_backend and not instance_name: if storage_backend and not instance_name:
instance_name = storage_backend instance_name = storage_backend
+1 -1
View File
@@ -426,7 +426,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
pass pass
# If this was a store item, ingest the clip into the same store. # If this was a store item, ingest the clip into the same store.
stored_store: Optional[str] = None stored_instance: Optional[str] = None
stored_hash: Optional[str] = None stored_hash: Optional[str] = None
stored_path: Optional[str] = None stored_path: Optional[str] = None
+26 -2
View File
@@ -74,8 +74,32 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
def provider_display_name(key: str) -> str: def provider_display_name(key: str) -> str:
label = (key or "").strip() label = (key or "").strip().lower()
return label[:1].upper() + label[1:] if label else "Plugin" if not label:
return "Plugin"
# Preserve expected brand casing for common providers.
display_overrides = {
"youtube": "YouTube",
"openlibrary": "OpenLibrary",
"podcastindex": "PodcastIndex",
}
if label in display_overrides:
return display_overrides[label]
return label[:1].upper() + label[1:]
def default_provider_ping_targets(key: str) -> list[str]:
"""Return default health-check URLs for known providers."""
label = (key or "").strip().lower()
defaults = {
"bandcamp": ["https://bandcamp.com"],
"youtube": ["https://www.youtube.com"],
"openlibrary": ["https://openlibrary.org"],
"podcastindex": ["https://podcastindex.org"],
"loc": ["https://www.loc.gov"],
}
return list(defaults.get(label, []))
def ping_first(urls: list[str]) -> tuple[bool, str]: def ping_first(urls: list[str]) -> tuple[bool, str]:
+5 -5
View File
@@ -19,7 +19,7 @@ The FTP plugin demonstrates the main provider hooks that matter for a storage-st
- `search()` walks an FTP directory tree and returns `SearchResult` rows. - `search()` walks an FTP directory tree and returns `SearchResult` rows.
- `selector()` turns folder rows into a follow-up table when the user runs `@N`. - `selector()` turns folder rows into a follow-up table when the user runs `@N`.
- `download()` and `download_url()` fetch FTP files into `download-file` output paths. - `download()` and `download_url()` fetch FTP files into `download-file` output paths.
- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first. - `resolve_pipe_result_download()` lets `@N | add-file -instance ...` materialize a remote FTP file first.
- `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server. - `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server.
## Example Config ## Example Config
@@ -114,11 +114,11 @@ search-file -plugin ftp -instance work "report"
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
``` ```
If you want to ingest the selected FTP file into a configured store backend: If you want to ingest the selected FTP file into a configured instance backend:
```powershell ```powershell
search-file -plugin ftp -instance work "report" search-file -plugin ftp -instance work "report"
@1 | add-file -store tutorial @1 | add-file -instance tutorial
``` ```
Why this works: Why this works:
@@ -150,7 +150,7 @@ That split is what keeps these two user experiences compatible:
- `@N` on a folder opens a new table - `@N` on a folder opens a new table
- `@N` on a file downloads the file - `@N` on a file downloads the file
- `@N | add-file -store ...` first downloads, then ingests - `@N | add-file -instance ...` first downloads, then ingests
## Implementation Notes ## Implementation Notes
@@ -169,6 +169,6 @@ search-file -plugin ftp -instance work "*"
search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf" search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf"
@1 @1
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
@1 | add-file -store tutorial @1 | add-file -instance tutorial
add-file -plugin ftp -instance archive -path C:\Media\report.pdf add-file -plugin ftp -instance archive -path C:\Media\report.pdf
``` ```
+1 -1
View File
@@ -9,7 +9,7 @@ Key ideas
Example: Example:
plugin-table -plugin example -sample | @1 | add-file -store default plugin-table -plugin example -sample | @1 | add-file -instance default
What plugins must implement What plugins must implement
- An adapter that yields `ResultModel` objects (breaking API). - An adapter that yields `ResultModel` objects (breaking API).
+4 -4
View File
@@ -14,7 +14,7 @@ The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
- `search-file -plugin scp -instance <name> ...` lists remote files and folders over SFTP. - `search-file -plugin scp -instance <name> ...` lists remote files and folders over SFTP.
- plain `@N` on a folder drills into that directory. - plain `@N` on a folder drills into that directory.
- plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`. - plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`.
- `@N | add-file -store ...` downloads first, then ingests the local temp file. - `@N | add-file -instance ...` downloads first, then ingests the local temp file.
- `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path. - `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path.
## Example Config ## Example Config
@@ -105,11 +105,11 @@ search-file -plugin scp -instance work "report"
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
``` ```
Ingest a selected remote file into a configured store backend: Ingest a selected remote file into a configured instance backend:
```powershell ```powershell
search-file -plugin scp -instance work "report" search-file -plugin scp -instance work "report"
@1 | add-file -store tutorial @1 | add-file -instance tutorial
``` ```
Why this works: Why this works:
@@ -141,6 +141,6 @@ search-file -plugin scp -instance work "*"
search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip" search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip"
@1 @1
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
@1 | add-file -store tutorial @1 | add-file -instance tutorial
add-file -plugin scp -instance archive -path C:\Media\report.pdf add-file -plugin scp -instance archive -path C:\Media\report.pdf
``` ```
+3 -3
View File
@@ -52,9 +52,9 @@ class MyPlugin(Provider):
Bundled walkthrough: Bundled walkthrough:
- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`; plugin-mode `-store <name>` remains a compatibility alias while store-backed flows still use real backend stores. - Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`.
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). - The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance <name>` uploads. - The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance <name>` uploads. - The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The provider now resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims. - The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The provider now resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims.
+3 -3
View File
@@ -210,8 +210,8 @@ def _get_debrid_api_key(config: Dict[str, Any]) -> Optional[str]:
api_key=... api_key=...
-> config["provider"]["alldebrid"]["api_key"] -> config["provider"]["alldebrid"]["api_key"]
- store-style debrid block: - plugin-style debrid block:
config["store"]["debrid"]["all-debrid"]["api_key"] config["plugin"]["debrid"]["all-debrid"]["api_key"]
Falls back to some legacy keys if present. Falls back to some legacy keys if present.
""" """
@@ -227,7 +227,7 @@ def _get_debrid_api_key(config: Dict[str, Any]) -> Optional[str]:
if isinstance(entry, str) and entry.strip(): if isinstance(entry, str) and entry.strip():
return entry.strip() return entry.strip()
# 2) store.debrid block (canonical for debrid store configuration) # 2) plugin debrid block
try: try:
from SYS.config import get_debrid_api_key from SYS.config import get_debrid_api_key
+1 -1
View File
@@ -8,7 +8,7 @@ Run this to see sample output:
python -m Provider.example_provider python -m Provider.example_provider
Example usage (piped selector): Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default plugin-table -plugin example -sample | select -select 1 | add-file -instance default
""" """
from __future__ import annotations from __future__ import annotations
+2
View File
@@ -72,6 +72,8 @@ def _unique_path(path: Path) -> Path:
class FTP(Provider): class FTP(Provider):
PLUGIN_NAME = "ftp" PLUGIN_NAME = "ftp"
URL = ("ftp://", "ftps://") URL = ("ftp://", "ftps://")
MULTI_INSTANCE = True
SUPPORTED_CMDLETS = frozenset({"add-file", "get-file", "search-file"})
@property @property
def label(self) -> str: def label(self) -> str:
+2
View File
@@ -301,6 +301,8 @@ class Matrix(TableProviderMixin, Provider):
""" """
EXPOSE_AS_FILE_PROVIDER = False EXPOSE_AS_FILE_PROVIDER = False
MULTI_INSTANCE = True
SUPPORTED_CMDLETS = frozenset({"add-file"})
@classmethod @classmethod
def config_schema(cls) -> List[Dict[str, Any]]: def config_schema(cls) -> List[Dict[str, Any]]:
+1 -1
View File
@@ -472,7 +472,7 @@ class MPV:
pipeline = f"download-file -url {_q(url)} -query {_q(f'format:{fmt}')}" pipeline = f"download-file -url {_q(url)} -query {_q(f'format:{fmt}')}"
if store: if store:
pipeline += f" | add-file -store {_q(store)}" pipeline += f" | add-file -instance {_q(store)}"
else: else:
pipeline += f" | add-file -path {_q(path or '')}" pipeline += f" | add-file -path {_q(path or '')}"
+1 -1
View File
@@ -30,7 +30,7 @@ class Vimm(TableProviderMixin, Provider):
- This provider exposes file rows on a detail page. Each file row includes - This provider exposes file rows on a detail page. Each file row includes
a `path` which is an absolute download URL (or a form action + mediaId). a `path` which is an absolute download URL (or a form action + mediaId).
- To make `@N` expansion robust (so users can do `@1 | add-file -store <x>`) - To make `@N` expansion robust (so users can do `@1 | add-file -instance <x>`)
we ensure three things: we ensure three things:
1) The ResultTable produced by the `selector()` sets `source_command` to 1) The ResultTable produced by the `selector()` sets `source_command` to
"download-file" (the canonical cmdlet for downloading files). "download-file" (the canonical cmdlet for downloading files).
+5 -5
View File
@@ -793,9 +793,9 @@ def main() -> int:
conn.row_factory = sqlite3.Row conn.row_factory = sqlite3.Row
cur = conn.cursor() cur = conn.cursor()
# Find all existing hydrusnetwork store names # Find all existing hydrusnetwork instance names
cur.execute( cur.execute(
"SELECT DISTINCT item_name FROM config WHERE category='store' AND subtype='hydrusnetwork'" "SELECT DISTINCT item_name FROM config WHERE category='plugin' AND subtype='hydrusnetwork'"
) )
rows = cur.fetchall() rows = cur.fetchall()
item_names = [r[0] for r in rows if r[0]] item_names = [r[0] for r in rows if r[0]]
@@ -804,12 +804,12 @@ def main() -> int:
# Only create if none exist. Use a sensible name from the path if possible. # Only create if none exist. Use a sensible name from the path if possible.
# We don't have the hydrus_path here easily, but we can try to find it. # We don't have the hydrus_path here easily, but we can try to find it.
# For now, if we are in bootstrap, we might just be setting a global. # For now, if we are in bootstrap, we might just be setting a global.
# But this function is specifically for store settings. # But this function is specifically for instance settings.
# Let's use 'home' instead of 'hydrus' as it's the standard default. # Let's use 'home' instead of 'hydrus' as it's the standard default.
item_name = "home" item_name = "home"
cur.execute( cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', item_name, 'NAME', item_name) ('plugin', 'hydrusnetwork', item_name, 'NAME', item_name)
) )
item_names = [item_name] item_names = [item_name]
@@ -817,7 +817,7 @@ def main() -> int:
for name in item_names: for name in item_names:
cur.execute( cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', name, key, value) ('plugin', 'hydrusnetwork', name, key, value)
) )
conn.commit() conn.commit()
+1 -1
View File
@@ -335,7 +335,7 @@ def main(argv: Optional[List[str]] = None) -> int:
# expand it into argv tokens (PowerShell commonly encourages quoting strings). # expand it into argv tokens (PowerShell commonly encourages quoting strings).
# #
# Examples: # Examples:
# mm "download-file <url> | add-tag 'x' | add-file -store local" # mm "download-file <url> | add-tag 'x' | add-file -instance local"
# mm "download-file '<url>' -query 'format:720' -path 'C:\\out'" # mm "download-file '<url>' -query 'format:720' -path 'C:\\out'"
if len(clean_args) == 1: if len(clean_args) == 1:
single = clean_args[0] single = clean_args[0]
+3 -3
View File
@@ -188,7 +188,7 @@ def update_medios_config(hydrus_path: Path) -> bool:
cur = conn.cursor() cur = conn.cursor()
cur.execute( cur.execute(
"SELECT DISTINCT item_name FROM config WHERE category='store' AND subtype='hydrusnetwork'" "SELECT DISTINCT item_name FROM config WHERE category='plugin' AND subtype='hydrusnetwork'"
) )
rows = [row[0] for row in cur.fetchall() if row[0]] rows = [row[0] for row in cur.fetchall() if row[0]]
@@ -196,14 +196,14 @@ def update_medios_config(hydrus_path: Path) -> bool:
store_name = _sanitize_store_name(hydrus_path.name) store_name = _sanitize_store_name(hydrus_path.name)
cur.execute( cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', store_name, 'name', store_name) ('plugin', 'hydrusnetwork', store_name, 'name', store_name)
) )
rows = [store_name] rows = [store_name]
for name in rows: for name in rows:
cur.execute( cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', name, 'gitclone', hydrus_abs_path) ('plugin', 'hydrusnetwork', name, 'gitclone', hydrus_abs_path)
) )
conn.commit() conn.commit()