diff --git a/CLI.py b/CLI.py index e4feb34..bcef5ad 100644 --- a/CLI.py +++ b/CLI.py @@ -506,17 +506,17 @@ class CmdletIntrospection: if normalized_arg == "plugin": canonical_cmd = (cmd_name or "").replace("_", "-").lower() try: - from ProviderCore.registry import list_plugin_names_with_capability + from ProviderCore.registry import list_configured_plugin_names_with_capability except Exception: - list_plugin_names_with_capability = None # type: ignore + list_configured_plugin_names_with_capability = None # type: ignore plugin_choices: List[str] = [] - if canonical_cmd in {"add-file"} and list_plugin_names_with_capability is not None: - return list_plugin_names_with_capability("upload") or [] + if canonical_cmd in {"add-file"} and list_configured_plugin_names_with_capability is not None: + return list_configured_plugin_names_with_capability("upload", config) or [] - if list_plugin_names_with_capability is not None: - plugin_choices = list_plugin_names_with_capability("search") or [] + if list_configured_plugin_names_with_capability is not None: + plugin_choices = list_configured_plugin_names_with_capability("search", config) or [] if plugin_choices: return plugin_choices diff --git a/ProviderCore/base.py b/ProviderCore/base.py index 03e6795..f1d2012 100644 --- a/ProviderCore/base.py +++ b/ProviderCore/base.py @@ -161,6 +161,17 @@ class Provider(ABC): # generic "file host" plugins via `add-file -plugin ...`. EXPOSE_AS_FILE_PROVIDER: bool = True + # Set to True for plugins that support multiple named instances in config. + # When True, config is expected at config["plugin"][][] + # rather than config["plugin"][] directly. + # Examples: hydrusnetwork (home/work), matrix (personal/work), ftp. + MULTI_INSTANCE: bool = False + + # Declare which top-level cmdlet names this plugin handles. + # Cmdlet dispatch and capability discovery use this to route operations. + # Example: frozenset({"add-file", "get-file", "get-tag", "search-file"}) + SUPPORTED_CMDLETS: frozenset = frozenset() + def __init__(self, config: Optional[Dict[str, Any]] = None): self.config = config or {} self.name = str( @@ -312,11 +323,21 @@ class Provider(ABC): def plugin_config_root(self) -> Dict[str, Any]: if not isinstance(self.config, dict): return {} - provider_cfg = self.config.get("provider") - if not isinstance(provider_cfg, dict): - return {} - entry = provider_cfg.get(self.plugin_config_key()) - return dict(entry) if isinstance(entry, dict) else {} + # Check plugin/provider section first (preferred new format) + for section in ("plugin", "provider"): + section_cfg = self.config.get(section) + if isinstance(section_cfg, dict): + entry = section_cfg.get(self.plugin_config_key()) + if isinstance(entry, dict): + return dict(entry) + # Backward compat: fall back to store section. + # store config uses {type: {instance: {key: val}}} — one level deeper. + store_cfg = self.config.get("store") + if isinstance(store_cfg, dict): + store_entries = store_cfg.get(self.plugin_config_key()) + if isinstance(store_entries, dict): + return dict(store_entries) + return {} def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]: entry = self.plugin_config_root() @@ -599,6 +620,70 @@ class Provider(ABC): """Upload a file and return a URL or identifier.""" raise NotImplementedError(f"Plugin '{self.name}' does not support upload") + # ----------------------------------------------------------------------- + # Storage interface — mirrors Store._base.Store. + # Plugins that act as file repositories override these methods. + # All raise NotImplementedError by default; override selectively. + # ----------------------------------------------------------------------- + + @property + def is_remote(self) -> bool: + """True if this plugin stores files on a remote service.""" + return False + + @property + def prefer_defer_tags(self) -> bool: + """True if tag writes should be deferred until after file ingest.""" + return False + + def add_file(self, file_path: Path, **kwargs: Any) -> str: + """Ingest a file and return its canonical hash.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support add_file") + + def get_file(self, file_hash: str, **kwargs: Any) -> Optional[Path]: + """Retrieve a stored file by hash, returning a local Path or None.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support get_file") + + def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: + """Return metadata dict for a stored file.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support get_metadata") + + def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]: + """Return (tags, hash) for a stored file identifier.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support get_tag") + + def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: + """Add tags to a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support add_tag") + + def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: + """Remove tags from a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support delete_tag") + + def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]: + """Return associated URLs for a stored file.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support get_url") + + def add_url(self, file_identifier: str, urls: List[str], **kwargs: Any) -> bool: + """Associate URLs with a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support add_url") + + def delete_url(self, file_identifier: str, urls: List[str], **kwargs: Any) -> bool: + """Remove URL associations from a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support delete_url") + + def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]: + """Return notes dict (name -> text) for a stored file.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support get_note") + + def set_note(self, file_identifier: str, name: str, text: str, **kwargs: Any) -> bool: + """Write a named note on a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support set_note") + + def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool: + """Delete a named note from a stored file. Returns True on success.""" + raise NotImplementedError(f"Plugin '{self.name}' does not support delete_note") + def validate(self) -> bool: """Check if the plugin is available and properly configured.""" diff --git a/ProviderCore/registry.py b/ProviderCore/registry.py index 0d896e3..a8863a1 100644 --- a/ProviderCore/registry.py +++ b/ProviderCore/registry.py @@ -150,6 +150,20 @@ class PluginInfo: exposed = True return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload) + @property + def is_multi_instance(self) -> bool: + """True if the plugin declares MULTI_INSTANCE = True.""" + return bool(getattr(self.plugin_class, "MULTI_INSTANCE", False)) + + @property + def supported_cmdlets(self) -> frozenset: + """Frozenset of cmdlet names this plugin declares support for.""" + raw = getattr(self.plugin_class, "SUPPORTED_CMDLETS", frozenset()) + try: + return frozenset(str(c) for c in raw) + except Exception: + return frozenset() + class PluginRegistry: """Handles discovery, registration, and lookup of built-in and external plugins.""" @@ -433,6 +447,42 @@ class PluginRegistry: def has_name(self, name: str) -> bool: return self.get(name) is not None + def get_plugins_for_cmdlet(self, cmdlet_name: str) -> List[PluginInfo]: + """Return all plugins that declare support for the given cmdlet name.""" + self.discover() + target = str(cmdlet_name or "").strip().lower() + return [ + info for info in self._infos.values() + if target in info.supported_cmdlets + ] + + def list_storage_plugin_instances( + self, + config: Optional[Dict[str, Any]] = None, + ) -> Dict[str, List[str]]: + """Return {plugin_name: [instance_name, ...]} for all MULTI_INSTANCE storage plugins. + + Instance names come from the plugin's resolved config (plugin/provider/store sections). + Plugins with no configured instances are omitted. + """ + self.discover() + result: Dict[str, List[str]] = {} + for info in self._infos.values(): + if not info.is_multi_instance: + continue + if not info.supported_cmdlets.intersection( + {"add-file", "get-file", "get-tag", "add-tag"} + ): + continue + try: + instance = info.plugin_class(config or {}) + instances = instance.configured_instances() + if instances: + result[info.canonical_name] = instances + except Exception: + pass + return result + def _sync_subclasses(self) -> None: """Walk all plugin subclasses in memory and register them.""" def _walk(cls: Type[Provider]) -> None: @@ -691,6 +741,39 @@ def list_plugin_names_with_capability(capability: str) -> List[str]: ) +def list_configured_plugin_names_with_capability( + capability: str, + config: Optional[Dict[str, Any]] = None, +) -> List[str]: + """Return plugin names that support `capability` AND have configuration present. + + For MULTI_INSTANCE plugins (e.g. hydrusnetwork, ftp) the plugin must have at + least one configured instance. For single-instance plugins the plugin's section + must exist under config["plugin"] or config["provider"]. + """ + cfg = config or {} + plugin_section: Dict[str, Any] = cfg.get("plugin") or {} # type: ignore[assignment] + provider_section: Dict[str, Any] = cfg.get("provider") or {} # type: ignore[assignment] + + result: List[str] = [] + for info in REGISTRY.iter_plugins(): + if not _info_supports_capability(info, capability): + continue + name = info.canonical_name + if info.is_multi_instance: + try: + instances = info.plugin_class(cfg).configured_instances() + if instances: + result.append(name) + except Exception: + pass + else: + pname = name.lower() + if isinstance(plugin_section.get(pname), dict) or isinstance(provider_section.get(pname), dict): + result.append(name) + return sorted(result) + + def match_plugin_name_for_url(url: str) -> Optional[str]: raw_url = str(url or "").strip() raw_url_lower = raw_url.lower() @@ -907,6 +990,7 @@ __all__ = [ "get_plugin_with_capability", "list_plugins_with_capability", "list_plugin_names_with_capability", + "list_configured_plugin_names_with_capability", "match_plugin_name_for_url", "get_plugin_for_url", "list_selection_url_prefixes", diff --git a/SYS/cli_syntax.py b/SYS/cli_syntax.py index b6fe1e0..ef5360c 100644 --- a/SYS/cli_syntax.py +++ b/SYS/cli_syntax.py @@ -138,7 +138,7 @@ def _validate_add_note_requires_add_file_order(raw: str) -> Optional[SyntaxError # If add-note occurs before any add-file stage, it must be explicitly targeted. if any(pos > i for pos in add_file_positions): has_hash = _has_flag(tokens, "-hash", "--hash") - has_store = _has_flag(tokens, "-store", "--store") + has_store = _has_flag(tokens, "-instance", "--instance") # Also accept explicit targeting via -query "store: hash: ...". query_val = _get_flag_value(tokens, "-query", "--query") diff --git a/SYS/config.py b/SYS/config.py index 79ff0da..05938ff 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -97,16 +97,27 @@ def _log_config_load_summary(config: Dict[str, Any]) -> None: plugin_block = config.get("plugin") if not isinstance(plugin_block, dict): plugin_block = config.get("provider") - provs = list(plugin_block.keys()) if isinstance(plugin_block, dict) else [] - stores = list(config.get("store", {}).keys()) if isinstance(config.get("store"), dict) else [] + if isinstance(plugin_block, dict): + # Count distinct plugin names; note multi-instance plugins appear once per name + plugin_names = list(plugin_block.keys()) + # Count total configured instances across all plugins + total_instances = sum( + len(v) if isinstance(v, dict) and all(isinstance(x, dict) for x in v.values()) else 1 + for v in plugin_block.values() + if isinstance(v, dict) + ) + else: + plugin_names, total_instances = [], 0 mtime = None try: mtime = datetime.datetime.fromtimestamp(db.db_path.stat().st_mtime, datetime.timezone.utc).isoformat().replace('+00:00', 'Z') except Exception: mtime = None + plugins_str = ', '.join(plugin_names[:10]) + ('...' if len(plugin_names) > 10 else '') summary = ( - f"Loaded config from {db.db_path.name}: plugins={len(provs)} ({', '.join(provs[:10])}{'...' if len(provs)>10 else ''}), " - f"stores={len(stores)} ({', '.join(stores[:10])}{'...' if len(stores)>10 else ''}), mtime={mtime}" + f"Loaded config from {db.db_path.name}: " + f"plugins={len(plugin_names)} ({plugins_str}), " + f"instances={total_instances}, mtime={mtime}" ) log(summary) except Exception: @@ -254,37 +265,37 @@ def set_nested_config_value( def get_hydrus_instance( config: Dict[str, Any], instance_name: str = "home" ) -> Optional[Dict[str, Any]]: - """Get a specific Hydrus instance config by name. + """Get a specific Hydrus instance config by name from plugin/provider config.""" + def _lookup_in(source: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if not isinstance(source, dict) or not source: + return None + instance = source.get(instance_name) + if isinstance(instance, dict): + return instance + target = str(instance_name or "").lower() + for name, conf in source.items(): + if isinstance(conf, dict) and str(name).lower() == target: + return conf + keys = sorted(source.keys()) + for key in keys: + if not str(key or "").startswith("new_"): + candidate = source.get(key) + if isinstance(candidate, dict): + return candidate + first_key = keys[0] if keys else None + candidate = source.get(first_key) if first_key else None + return candidate if isinstance(candidate, dict) else None - Supports modern config plus a fallback when no exact match exists. - """ - store = config.get("store", {}) - if not isinstance(store, dict): - return None - - hydrusnetwork = store.get("hydrusnetwork", {}) - if not isinstance(hydrusnetwork, dict) or not hydrusnetwork: - return None - - instance = hydrusnetwork.get(instance_name) - if isinstance(instance, dict): - return instance - - target = str(instance_name or "").lower() - for name, conf in hydrusnetwork.items(): - if isinstance(conf, dict) and str(name).lower() == target: - return conf - - keys = sorted(hydrusnetwork.keys()) - for key in keys: - if not str(key or "").startswith("new_"): - candidate = hydrusnetwork.get(key) - if isinstance(candidate, dict): - return candidate - first_key = keys[0] - candidate = hydrusnetwork.get(first_key) - if isinstance(candidate, dict): - return candidate + # New format: config["plugin"]["hydrusnetwork"] or config["provider"]["hydrusnetwork"] + # (both point to the same dict after normalization) + for section in ("plugin", "provider"): + section_cfg = config.get(section) + if isinstance(section_cfg, dict): + hydrus_cfg = section_cfg.get("hydrusnetwork") + if isinstance(hydrus_cfg, dict): + result = _lookup_in(hydrus_cfg) + if result is not None: + return result return None @@ -293,7 +304,7 @@ def get_hydrus_access_key(config: Dict[str, Any], instance_name: str = "home") - """Get Hydrus access key for an instance. Config format: - - config["store"]["hydrusnetwork"][name]["API"] + - config["plugin"]["hydrusnetwork"][name]["API"] Args: config: Configuration dict @@ -314,7 +325,7 @@ def get_hydrus_url(config: Dict[str, Any], instance_name: str = "home") -> Optio """Get Hydrus URL for an instance. Config format: - - config["store"]["hydrusnetwork"][name]["URL"] + - config["plugin"]["hydrusnetwork"][name]["URL"] Args: config: Configuration dict @@ -438,9 +449,7 @@ def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]: def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> Optional[str]: """Get Debrid API key from config. - Config format: - - config["store"]["debrid"][]["api_key"] - where is the store name (e.g. "all-debrid") + Checks the plugin/provider block first (canonical format). Args: config: Configuration dict @@ -449,23 +458,27 @@ def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> O Returns: API key string if found, None otherwise """ - store = config.get("store", {}) - if not isinstance(store, dict): - return None + # 1) Canonical plugin/provider block: config["plugin"]["alldebrid"]["api_key"] + provider_block = config.get("provider") or config.get("plugin") + if isinstance(provider_block, dict): + alldebrid_entry = provider_block.get("alldebrid") + if isinstance(alldebrid_entry, dict): + for k in ("api_key", "API_KEY", "apikey", "APIKEY"): + val = alldebrid_entry.get(k) + if isinstance(val, str) and val.strip(): + return val.strip() - debrid_config = store.get("debrid", {}) - if not isinstance(debrid_config, dict): - return None - - service_key = str(service).strip().lower() - entry = debrid_config.get(service_key) - - if isinstance(entry, dict): - api_key = entry.get("api_key") - return str(api_key).strip() if api_key else None - - if isinstance(entry, str): - return entry.strip() or None + # 2) Migrated legacy debrid plugin entry: config["plugin"]["debrid"]["all-debrid"]["api_key"] + if isinstance(provider_block, dict): + service_key = str(service).strip().lower() + debrid_plugin = provider_block.get("debrid") + if isinstance(debrid_plugin, dict): + entry = debrid_plugin.get(service_key) + if isinstance(entry, dict): + api_key = entry.get("api_key") + return str(api_key).strip() if api_key else None + if isinstance(entry, str): + return entry.strip() or None return None @@ -635,6 +648,25 @@ def _normalize_plugin_config_aliases(config: Dict[str, Any]) -> None: if normalized_key and normalized_key not in normalized_provider: normalized_provider[normalized_key] = value + # Fold legacy config["store"] entries into the plugin namespace. + # store format: {type: {instance_name: {key: val}}} — multi-instance. + # After folding, remove config["store"] so it is no longer consulted. + store_block = config.pop("store", None) + if isinstance(store_block, dict): + for store_type, instances in store_block.items(): + if not isinstance(instances, dict): + continue + normalized_key = _normalize_provider_name(store_type) + if not normalized_key: + continue + existing = normalized_provider.get(normalized_key) + if not isinstance(existing, dict): + existing = {} + normalized_provider[normalized_key] = existing + for instance_name, settings in instances.items(): + if isinstance(settings, dict) and instance_name not in existing: + existing[instance_name] = dict(settings) + if normalized_provider: config["provider"] = normalized_provider config["plugin"] = normalized_provider @@ -658,6 +690,11 @@ def _extract_api_key(value: Any) -> Optional[str]: def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None: + """Ensure AllDebrid API key is consistently stored in config[\"plugin\"][\"alldebrid\"]. + + Previously this function also synced to config[\"store\"][\"debrid\"]. That path + is no longer used; only the plugin namespace is written. + """ if not isinstance(config, dict): return @@ -680,38 +717,39 @@ def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None: provider_section = {"api_key": provider_key} providers["alldebrid"] = provider_section - store_block = config.get("store") - if not isinstance(store_block, dict): - store_block = {} - config["store"] = store_block + # If no key found in provider block, check for a migrated debrid plugin entry. + # (rows_to_config migrates store.debrid.all-debrid → plugin.debrid.all-debrid) + if not provider_key: + plugin_block = config.get("plugin") or providers + debrid_plugin = plugin_block.get("debrid") if isinstance(plugin_block, dict) else None + if isinstance(debrid_plugin, dict): + service_entry = debrid_plugin.get("all-debrid") + legacy_key = _extract_api_key(service_entry) if service_entry else None + if legacy_key: + if provider_section is None: + provider_section = {} + providers["alldebrid"] = provider_section + provider_section.setdefault("api_key", legacy_key) - debrid_block = store_block.get("debrid") - store_key = None - if isinstance(debrid_block, dict): - service_entry = debrid_block.get("all-debrid") - if isinstance(service_entry, dict): - store_key = _extract_api_key(service_entry) - elif isinstance(service_entry, str): - store_key = service_entry.strip() - if store_key: - debrid_block["all-debrid"] = {"api_key": store_key} - else: - debrid_block = None - if provider_key: - if debrid_block is None: - debrid_block = {} - store_block["debrid"] = debrid_block - service_section = debrid_block.get("all-debrid") - if not isinstance(service_section, dict): - service_section = {} - debrid_block["all-debrid"] = service_section - service_section["api_key"] = provider_key - elif store_key: - if provider_section is None: - provider_section = {} - providers["alldebrid"] = provider_section - provider_section["api_key"] = store_key + +def _is_multi_instance_plugin_config(value: Any) -> bool: + """Return True if `value` looks like a multi-instance plugin config (dict-of-dicts). + + Multi-instance plugins store their configuration as:: + + {: {key: value, ...}, ...} + + Single-instance plugins store their config as a flat dict:: + + {key: value, ...} + + We detect multi-instance by checking whether ALL values are themselves dicts + (and the outer dict is non-empty). An empty dict is treated as single-instance. + """ + if not isinstance(value, dict) or not value: + return False + return all(isinstance(v, dict) for v in value.values()) def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str, str], Any]: @@ -719,18 +757,35 @@ def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str, _normalize_plugin_config_aliases(config) for key, value in config.items(): if key == 'plugin': + # plugin == provider after normalization; skip duplicate continue - if key in ('store', 'provider', 'tool') and isinstance(value, dict): + if key == 'provider' and isinstance(value, dict): + for subtype, plugin_cfg in value.items(): + if not isinstance(plugin_cfg, dict): + continue + if _is_multi_instance_plugin_config(plugin_cfg): + # Multi-instance: {instance_name: {key: val}} + for instance_name, settings in plugin_cfg.items(): + if not isinstance(settings, dict): + continue + for k, v in settings.items(): + entries[('plugin', subtype, instance_name, k)] = v + else: + # Single-instance: {key: val} + for k, v in plugin_cfg.items(): + entries[('provider', subtype, 'default', k)] = v + elif key in ('store', 'tool') and isinstance(value, dict): for subtype, instances in value.items(): if not isinstance(instances, dict): continue if key == 'store': + # Legacy store: migrate to plugin category for name, settings in instances.items(): if not isinstance(settings, dict): continue for k, v in settings.items(): - entries[(key, subtype, name, k)] = v - else: + entries[('plugin', subtype, name, k)] = v + else: # tool for k, v in instances.items(): entries[(key, subtype, 'default', k)] = v elif not key.startswith('_') and value is not None: @@ -763,12 +818,23 @@ def _config_from_flattened_entries( continue if category == "store": - store_block = config.setdefault("store", {}) - subtype_block = store_block.setdefault(subtype, {}) + # Legacy: migrate to plugin namespace at reconstitution time + plugin_block = config.setdefault("plugin", {}) + subtype_block = plugin_block.setdefault(subtype, {}) item_block = subtype_block.setdefault(item_name, {}) item_block[key] = value continue + if category == "plugin": + plugin_block = config.setdefault("plugin", {}) + subtype_block = plugin_block.setdefault(subtype, {}) + if item_name == "default": + subtype_block[key] = value + else: + item_block = subtype_block.setdefault(item_name, {}) + item_block[key] = value + continue + if category in {"provider", "tool"}: category_block = config.setdefault(category, {}) subtype_block = category_block.setdefault(subtype, {}) @@ -827,18 +893,7 @@ def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]: elif isinstance(entry, str) and entry.strip(): expected_key = entry.strip() if not expected_key: - store_block = config.get("store", {}) if isinstance(config, dict) else {} - debrid = store_block.get("debrid") if isinstance(store_block, dict) else None - if isinstance(debrid, dict): - srv = debrid.get("all-debrid") - if isinstance(srv, dict): - for k in ("api_key", "API_KEY", "apikey", "APIKEY"): - v = srv.get(k) - if isinstance(v, str) and v.strip(): - expected_key = v.strip() - break - elif isinstance(srv, str) and srv.strip(): - expected_key = srv.strip() + expected_key = get_debrid_api_key(config, service="All-debrid") except Exception as exc: logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True) expected_key = None @@ -853,6 +908,14 @@ def load_config(*, emit_summary: bool = True) -> Dict[str, Any]: _CONFIG_SUMMARY_PENDING = False return _CONFIG_CACHE + # One-time DB migration: move category='store' rows to category='plugin'. + # This is idempotent — a no-op if no store rows exist. + try: + from SYS.database import migrate_store_category_to_plugin + migrate_store_category_to_plugin() + except Exception: + logger.debug("Store→plugin DB migration skipped or failed", exc_info=True) + # Load strictly from database db_config = get_config_all() if db_config: diff --git a/SYS/database.py b/SYS/database.py index 7cea074..846eb0c 100644 --- a/SYS/database.py +++ b/SYS/database.py @@ -510,7 +510,10 @@ def rows_to_config(rows) -> Dict[str, Any]: sub_dict = cat_dict.setdefault(sub, {}) sub_dict[key] = parsed_val elif cat == 'store': - cat_dict = config.setdefault(cat, {}) + # Migrate legacy store rows into the unified plugin namespace. + # store config used a 4-level path: (store, type, instance_name, key). + # Plugin config uses: config["plugin"][type][instance_name][key]. + cat_dict = config.setdefault('plugin', {}) sub_dict = cat_dict.setdefault(sub, {}) name_dict = sub_dict.setdefault(name, {}) name_dict[key] = parsed_val @@ -520,11 +523,52 @@ def rows_to_config(rows) -> Dict[str, Any]: return config +def migrate_store_category_to_plugin() -> int: + """One-time migration: re-key category='store' DB rows to category='plugin'. + + The 'store' category used ``(store, type, instance_name, key)`` tuples; + the unified plugin system uses the same 4-level path under category='plugin'. + Existing 'plugin' rows for the same (subtype, item_name, key) are overwritten. + + Returns the number of rows that were migrated (0 if already migrated). + """ + try: + count_row = db.fetchone( + "SELECT COUNT(*) AS n FROM config WHERE category='store' AND subtype != 'folder'" + ) + count = int(count_row['n']) if count_row else 0 + if count == 0: + # Also clean up any lingering folder-store rows + db.execute("DELETE FROM config WHERE category='store'") + with db._conn_lock: + db.conn.commit() + return 0 + # Copy store rows to plugin, replacing any pre-existing plugin rows for + # the same (subtype, item_name, key), then delete the old store rows. + db.execute( + """ + INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) + SELECT 'plugin', subtype, item_name, key, value + FROM config + WHERE category = 'store' AND subtype != 'folder' + """ + ) + db.execute("DELETE FROM config WHERE category = 'store'") + with db._conn_lock: + db.conn.commit() + logger.info("Migrated %d config rows from category='store' to category='plugin'", count) + return count + except Exception: + logger.exception("Failed to migrate store config rows to plugin category") + return 0 + + def get_config_all() -> Dict[str, Any]: """Retrieve all configuration from the database in the legacy dict format.""" rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config") return rows_to_config(rows) + # Worker Management Methods for medios.db def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection: diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 93c7208..f42f967 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1272,7 +1272,7 @@ class PipelineExecutor: """Guard against running add-relationship on unstored download-file results. Intended UX: - download-file ... | add-file -store | add-relationship + download-file ... | add-file -instance | add-relationship Rationale: download-file outputs items that may not yet have a stable store+hash. @@ -1305,7 +1305,7 @@ class PipelineExecutor: print( "Pipeline order error: when using download-file with add-relationship, " "add-relationship must come after add-file (so items are stored and have store+hash).\n" - "Example: download-file <...> | add-file -store | add-relationship\n" + "Example: download-file <...> | add-file -instance | add-relationship\n" ) return False diff --git a/SYS/plugin_config.py b/SYS/plugin_config.py index 6129fa0..6679f13 100644 --- a/SYS/plugin_config.py +++ b/SYS/plugin_config.py @@ -7,7 +7,6 @@ from typing import Any, Dict, Iterable, List, Optional from SYS.config import global_config from ProviderCore.registry import get_plugin_class, list_plugins -from Store.registry import _discover_store_classes, _required_keys_for, _resolve_store_class logger = logging.getLogger(__name__) @@ -54,10 +53,16 @@ def _call_schema(owner: Any, label: str) -> List[ConfigField]: def get_store_schema(store_type: str) -> List[ConfigField]: - cls = _resolve_store_class(str(store_type or "").strip()) - if cls is None: - return [] - return _call_schema(cls, f"store '{store_type}'") + """Return config schema for a store type. + + After the store→plugin migration, store types are plugins. We look up the + plugin schema by name; if not found we return an empty list. + """ + normalized = str(store_type or "").strip() + # Strip a legacy "store-" prefix so callers using the old type name still work + if normalized.startswith("store-"): + normalized = normalized[len("store-"):] + return get_plugin_schema(normalized) def get_plugin_schema(plugin_name: str) -> List[ConfigField]: @@ -84,6 +89,10 @@ def get_item_schema(item_type: str, item_name: str) -> List[ConfigField]: normalized_name = str(item_name or "").strip() if normalized_type.startswith("store-"): return get_store_schema(normalized_type.replace("store-", "", 1)) + if normalized_type.startswith("plugin-"): + # Multi-instance plugin: plugin-{ptype}; item_name is the instance name + ptype = normalized_type[len("plugin-"):] + return get_plugin_schema(ptype) if normalized_type in {"provider", "plugin"}: return get_plugin_schema(normalized_name) if normalized_type == "tool": @@ -104,23 +113,14 @@ def get_global_schema_map() -> Dict[str, ConfigField]: def build_default_store_config(store_type: str, instance_name: str) -> Dict[str, Any]: + """Build a default config dict for a new store/multi-instance plugin entry.""" config: Dict[str, Any] = {"NAME": instance_name} schema = get_store_schema(store_type) - if schema: - for field in schema: - key = field["key"] - if key.upper() == "NAME": - continue - config[key] = field.get("default", "") - return config - - cls = _resolve_store_class(str(store_type or "").strip()) - if cls is None: - return config - for required_key in _required_keys_for(cls): - if required_key.upper() == "NAME": + for field in schema: + key = field["key"] + if key.upper() == "NAME": continue - config[required_key] = "" + config[key] = field.get("default", "") return config @@ -170,12 +170,16 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]: if field.get("required"): _add_key(field.get("key")) - if normalized_type.startswith("store-"): - store_type = normalized_type.replace("store-", "", 1) - cls = _resolve_store_class(store_type) - if cls is not None: - for required_key in _required_keys_for(cls): - _add_key(required_key) + if normalized_type.startswith("plugin-") or normalized_type.startswith("store-"): + # Multi-instance plugin (plugin-{ptype}) or legacy store-{type}: look up by plugin name + ptype = normalized_type.replace("plugin-", "", 1).replace("store-", "", 1) + plugin_class = get_plugin_class(ptype) + if plugin_class is not None: + try: + for required_key in plugin_class.required_config_keys(): + _add_key(required_key) + except Exception: + logger.exception("Failed to load required config keys for plugin '%s'", ptype) elif normalized_type in {"provider", "plugin"}: plugin_class = get_plugin_class(normalized_name) if plugin_class is not None: @@ -189,18 +193,24 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]: def get_configurable_store_types() -> List[str]: + """Return configurable multi-instance plugin types (formerly 'store types').""" + from ProviderCore.registry import REGISTRY options: List[str] = [] - for store_type in _discover_store_classes().keys(): - if get_store_schema(store_type): - options.append(str(store_type)) + for info in REGISTRY.iter_plugins(): + plugin_cls = info.plugin_class + if getattr(plugin_cls, 'MULTI_INSTANCE', False) and get_plugin_schema(info.canonical_name): + options.append(info.canonical_name) return sorted(set(options)) def get_configurable_plugin_types() -> List[str]: + """Return all plugin types that can be configured: those with a schema or MULTI_INSTANCE flag.""" + from ProviderCore.registry import REGISTRY options: List[str] = [] - for plugin_name in list_plugins().keys(): - if get_plugin_schema(plugin_name): - options.append(str(plugin_name)) + for info in REGISTRY.iter_plugins(): + plugin_cls = info.plugin_class + if get_plugin_schema(info.canonical_name) or getattr(plugin_cls, 'MULTI_INSTANCE', False): + options.append(info.canonical_name) return sorted(set(options)) diff --git a/SYS/rich_display.py b/SYS/rich_display.py index dbf4ab6..3fe7122 100644 --- a/SYS/rich_display.py +++ b/SYS/rich_display.py @@ -92,6 +92,8 @@ def show_plugin_config_panel( """Show a Rich panel explaining how to configure plugins.""" from rich.table import Table as RichTable from rich.console import Group + from rich.panel import Panel + from rich.text import Text if isinstance(plugin_names, str): plugins = [p.strip() for p in plugin_names.split(",")] @@ -127,6 +129,8 @@ def show_store_config_panel( """Show a Rich panel explaining how to configure storage backends.""" from rich.table import Table as RichTable from rich.console import Group + from rich.panel import Panel + from rich.text import Text if isinstance(store_names, str): stores = [s.strip() for s in store_names.split(",")] @@ -160,6 +164,8 @@ def show_available_plugins_panel(plugin_names: List[str]) -> None: """Show a Rich panel listing available/configured plugins.""" from rich.columns import Columns from rich.console import Group + from rich.panel import Panel + from rich.text import Text if not plugin_names: return diff --git a/SYS/selection_builder.py b/SYS/selection_builder.py index 946c6a2..2670eb3 100644 --- a/SYS/selection_builder.py +++ b/SYS/selection_builder.py @@ -45,7 +45,7 @@ def build_hash_store_selection( store_text = str(store_value or "").strip() if not hash_text or not store_text: return None, None - args = ["-query", f"hash:{hash_text}", "-store", store_text] + args = ["-query", f"hash:{hash_text}", "-instance", store_text] return args, [action_name] + list(args) diff --git a/TUI.py b/TUI.py index 46bf6fc..5882030 100644 --- a/TUI.py +++ b/TUI.py @@ -364,7 +364,7 @@ class TagEditorPopup(ModalScreen[None]): if to_del: del_args = " ".join(json.dumps(t) for t in to_del) - del_cmd = f"delete-tag -store {store_tok}{query_chunk} {del_args}" + del_cmd = f"delete-tag -instance {store_tok}{query_chunk} {del_args}" _log_pipeline_command("delete-tag", del_cmd) del_res = runner.run_pipeline(del_cmd, seeds=self._seeds, isolate=True) _log_pipeline_result("delete-tag", del_res) @@ -381,7 +381,7 @@ class TagEditorPopup(ModalScreen[None]): if to_add: add_args = " ".join(json.dumps(t) for t in to_add) - add_cmd = f"add-tag -store {store_tok}{query_chunk} {add_args}" + add_cmd = f"add-tag -instance {store_tok}{query_chunk} {add_args}" _log_pipeline_command("add-tag", add_cmd) add_res = runner.run_pipeline(add_cmd, seeds=self._seeds, isolate=True) _log_pipeline_result("add-tag", add_res) @@ -1028,7 +1028,7 @@ class PipelineHubApp(App): Rules (simple + non-destructive): - If output path is set and the first stage is download-file and has no -path/--path, append -path. - - If a store is selected and pipeline has no add-file stage, append add-file -store . + - If an instance is selected and pipeline has no add-file stage, append add-file -instance . """ base = str(pipeline_text or "").strip() if not base: @@ -1080,7 +1080,7 @@ class PipelineHubApp(App): if should_auto_add_file: store_token = json.dumps(selected_store) - joined = f"{joined} | add-file -store {store_token}" + joined = f"{joined} | add-file -instance {store_token}" return joined @@ -1656,7 +1656,7 @@ class PipelineHubApp(App): try: if to_del: del_args = " ".join(json.dumps(t) for t in to_del) - del_cmd = f"delete-tag -store {store_tok}{query_chunk} {del_args}" + del_cmd = f"delete-tag -instance {store_tok}{query_chunk} {del_args}" del_res = runner.run_pipeline(del_cmd, seeds=seeds, isolate=True) if not getattr(del_res, "success", False): failures.append( @@ -1669,7 +1669,7 @@ class PipelineHubApp(App): if to_add: add_args = " ".join(json.dumps(t) for t in to_add) - add_cmd = f"add-tag -store {store_tok}{query_chunk} {add_args}" + add_cmd = f"add-tag -instance {store_tok}{query_chunk} {add_args}" add_res = runner.run_pipeline(add_cmd, seeds=seeds, isolate=True) if not getattr(add_res, "success", False): failures.append( @@ -2358,7 +2358,7 @@ class PipelineHubApp(App): self.notify("Delete action requires store + hash", severity="warning", timeout=3) return query = f"hash:{hash_value}" - cmd = f"delete-file -store {json.dumps(store_name)} -query {json.dumps(query)}" + cmd = f"delete-file -instance {json.dumps(store_name)} -query {json.dumps(query)}" self._start_pipeline_execution(cmd) return @@ -2398,11 +2398,11 @@ class PipelineHubApp(App): query = f"hash:{hash_value}" base_copy = ( - f"search-file -store {json.dumps(store_name)} {json.dumps(query)}" - f" | add-file -store {json.dumps(selected_store)}" + f"search-file -instance {json.dumps(store_name)} {json.dumps(query)}" + f" | add-file -instance {json.dumps(selected_store)}" ) if action == "move_to_selected_store": - delete_cmd = f"delete-file -store {json.dumps(store_name)} -query {json.dumps(query)}" + delete_cmd = f"delete-file -instance {json.dumps(store_name)} -query {json.dumps(query)}" cmd = f"{base_copy} | @ | {delete_cmd}" else: cmd = base_copy diff --git a/TUI/menu_actions.py b/TUI/menu_actions.py index 629700e..60d5615 100644 --- a/TUI/menu_actions.py +++ b/TUI/menu_actions.py @@ -32,13 +32,13 @@ PIPELINE_PRESETS: List[PipelinePreset] = [ description= "Use download-file with playlist auto-selection, merge the pieces, tag, then import into local storage.", pipeline= - 'download-file "" | merge-file | add-tags -store local | add-file -storage local', + 'download-file "" | merge-file | add-tags -instance local | add-file -storage local', ), PipelinePreset( label="Download → Hydrus", description="Fetch media, auto-tag, and push directly into Hydrus.", pipeline= - 'download-file "" | merge-file | add-tags -store hydrus | add-file -storage hydrus', + 'download-file "" | merge-file | add-tags -instance hydrus | add-file -storage hydrus', ), PipelinePreset( label="Search Local Library", diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index f6d0251..d527fe1 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -18,6 +18,7 @@ from SYS.config import ( count_changed_entries, ConfigSaveConflict, coerce_config_value, + _is_multi_instance_plugin_config, ) from SYS.database import db from SYS.logger import log, debug @@ -200,7 +201,6 @@ class ConfigModal(ModalScreen): yield Label("Categories", classes="config-label") with ListView(id="category-list"): yield ListItem(Label("Global Settings"), id="cat-globals") - yield ListItem(Label("Stores"), id="cat-stores") yield ListItem(Label("Plugins"), id="cat-providers") yield ListItem(Label("Tools"), id="cat-tools") @@ -210,14 +210,12 @@ class ConfigModal(ModalScreen): yield Button("Save", variant="success", id="save-btn") # Durable synchronous save: waits and verifies DB persisted critical keys yield Button("Save (durable)", variant="primary", id="save-durable-btn") - yield Button("Add Store", variant="primary", id="add-store-btn") yield Button("Add Plugin", variant="primary", id="add-provider-btn") yield Button("Add Tool", variant="primary", id="add-tool-btn") yield Button("Back", id="back-btn") yield Button("Close", variant="error", id="cancel-btn") def on_mount(self) -> None: - self.query_one("#add-store-btn", Button).display = False self.query_one("#add-provider-btn", Button).display = False try: self.query_one("#add-tool-btn", Button).display = False @@ -267,7 +265,6 @@ class ConfigModal(ModalScreen): # Update visibility of buttons try: - self.query_one("#add-store-btn", Button).display = (self.current_category == "stores" and self.editing_item_name is None) self.query_one("#add-provider-btn", Button).display = (self.current_category == "providers" and self.editing_item_name is None) self.query_one("#add-tool-btn", Button).display = (self.current_category == "tools" and self.editing_item_name is None) self.query_one("#back-btn", Button).display = (self.editing_item_name is not None) @@ -467,20 +464,46 @@ class ConfigModal(ModalScreen): providers = self.config_data.get("provider", {}) if not providers: container.mount(Static("No plugins configured.")) - else: - for i, (name, _) in enumerate(providers.items()): - edit_id = f"edit-provider-{i}" - del_id = f"del-provider-{i}" - self._button_id_map[edit_id] = ("edit", "provider", name) - self._button_id_map[del_id] = ("del", "provider", name) + return + idx = 0 + for plugin_name, plugin_cfg in providers.items(): + if isinstance(plugin_cfg, dict) and _is_multi_instance_plugin_config(plugin_cfg): + # Multi-instance plugin: show each instance as a separate row + for instance_name, instance_cfg in plugin_cfg.items(): + display_name = instance_name + if isinstance(instance_cfg, dict): + display_name = ( + instance_cfg.get("NAME") + or instance_cfg.get("name") + or instance_name + ) + edit_id = f"edit-provider-{idx}" + del_id = f"del-provider-{idx}" + self._button_id_map[edit_id] = ("edit", f"plugin-{plugin_name}", instance_name) + self._button_id_map[del_id] = ("del", f"plugin-{plugin_name}", instance_name) + row = Horizontal( + Static(f"{display_name} ({plugin_name})", classes="item-label"), + Button("Edit", id=edit_id), + Button("Delete", variant="error", id=del_id), + classes="item-row" + ) + container.mount(row) + idx += 1 + else: + # Single-instance plugin + edit_id = f"edit-provider-{idx}" + del_id = f"del-provider-{idx}" + self._button_id_map[edit_id] = ("edit", "plugin", plugin_name) + self._button_id_map[del_id] = ("del", "plugin", plugin_name) row = Horizontal( - Static(name, classes="item-label"), + Static(plugin_name, classes="item-label"), Button("Edit", id=edit_id), Button("Delete", variant="error", id=del_id), classes="item-row" ) container.mount(row) + idx += 1 def render_tools(self, container: ScrollableContainer) -> None: container.mount(Label("Configured Tools", classes="config-label")) @@ -508,11 +531,13 @@ class ConfigModal(ModalScreen): item_schema_map = get_item_schema_map(item_type, item_name) render_state = {"group": None, "mounted_any": False} - # Parse item_type for store-{stype} or just provider - if item_type.startswith("store-"): - stype = item_type.replace("store-", "") - container.mount(Label(f"Editing Store: {item_name} ({stype})", classes="config-label")) - section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {}) + # Parse item_type: plugin-{ptype} (multi-instance) or flat type + if item_type.startswith("plugin-"): + ptype = item_type[len("plugin-"):] + container.mount(Label(f"Editing {ptype}: {item_name}", classes="config-label")) + plugin_block = self.config_data.get("plugin") or self.config_data.get("provider") or {} + plugin_instances = plugin_block.get(ptype, {}) if isinstance(plugin_block, dict) else {} + section = plugin_instances.get(item_name, {}) if isinstance(plugin_instances, dict) else {} else: container.mount(Label(f"Editing {item_type.capitalize()}: {item_name}", classes="config-label")) section = self.config_data.get(item_type, {}).get(item_name, {}) @@ -598,7 +623,7 @@ class ConfigModal(ModalScreen): row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) idx += 1 - if item_type == "plugin" and isinstance(item_name, str): + if item_type in ("plugin", "provider") and isinstance(item_name, str): provider = self._instantiate_plugin_for_editor(item_name, self.config_data) if provider is not None: provider_actions = provider.config_actions() or [] @@ -626,7 +651,7 @@ class ConfigModal(ModalScreen): ) if ( - item_type == "plugin" + item_type in ("plugin", "provider") and isinstance(item_name, str) and item_name.strip().lower() == "matrix" ): @@ -720,13 +745,11 @@ class ConfigModal(ModalScreen): if not event.item: return item_id = getattr(event.item, "id", None) - if item_id not in ("cat-globals", "cat-stores", "cat-providers", "cat-tools"): + if item_id not in ("cat-globals", "cat-providers", "cat-tools"): return if item_id == "cat-globals": self.current_category = "globals" - elif item_id == "cat-stores": - self.current_category = "stores" elif item_id == "cat-providers": self.current_category = "providers" elif item_id == "cat-tools": @@ -841,13 +864,23 @@ class ConfigModal(ModalScreen): self.refresh_view() elif action == "del": removed = False - if itype.startswith("store-"): + if itype.startswith("plugin-"): + ptype = itype[len("plugin-"):] + plugin_block = self.config_data.get("plugin") or self.config_data.get("provider") + if isinstance(plugin_block, dict): + instances = plugin_block.get(ptype) + if isinstance(instances, dict) and name in instances: + del instances[name] + if not instances: + plugin_block.pop(ptype, None) + removed = True + elif itype.startswith("store-"): stype = itype.replace("store-", "") if "store" in self.config_data and stype in self.config_data["store"]: if name in self.config_data["store"][stype]: del self.config_data["store"][stype][name] removed = True - elif itype == "provider": + elif itype in ("provider", "plugin"): if "provider" in self.config_data and name in self.config_data["provider"]: del self.config_data["provider"][name] removed = True @@ -871,9 +904,6 @@ class ConfigModal(ModalScreen): elif bid in self._provider_button_map: provider_name, action_id = self._provider_button_map[bid] self._request_plugin_action(provider_name, action_id) - elif bid == "add-store-btn": - options = get_configurable_store_types() - self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected) elif bid == "add-provider-btn": options = get_configurable_plugin_types() self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected) @@ -1036,51 +1066,44 @@ class ConfigModal(ModalScreen): # Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed. - def on_store_type_selected(self, stype: str) -> None: - if not stype: - return - - self._capture_editor_snapshot() - - existing_names: set[str] = set() - store_block = self.config_data.get("store") - if isinstance(store_block, dict): - st_entries = store_block.get(stype) - if isinstance(st_entries, dict): - existing_names = {str(name) for name in st_entries.keys() if name} - - base_name = f"new_{stype}" - new_name = base_name - suffix = 1 - while new_name in existing_names: - suffix += 1 - new_name = f"{base_name}_{suffix}" - - if "store" not in self.config_data: - self.config_data["store"] = {} - if stype not in self.config_data["store"]: - self.config_data["store"][stype] = {} - - # Default config for the new store - new_config = build_default_store_config(stype, new_name) - - self.config_data["store"][stype][new_name] = new_config - self.editing_item_type = f"store-{stype}" - self.editing_item_name = new_name - self.refresh_view() - def on_provider_type_selected(self, ptype: str) -> None: - if not ptype: return + if not ptype: + return self._capture_editor_snapshot() - if "provider" not in self.config_data: - self.config_data["provider"] = {} - - # Plugins are configured under the top-level 'provider' dict for now. - if ptype not in self.config_data["provider"]: - self.config_data["provider"][ptype] = build_default_plugin_config(ptype) - - self.editing_item_type = "plugin" - self.editing_item_name = ptype + + from ProviderCore.registry import get_plugin_class as _get_cls + plugin_class = _get_cls(ptype) + is_multi = bool(getattr(plugin_class, 'MULTI_INSTANCE', False)) if plugin_class else False + + if is_multi: + # Multi-instance plugin: create a named instance entry in config["plugin"][ptype] + plugin_block = self.config_data.setdefault("plugin", {}) + instances = plugin_block.setdefault(ptype, {}) + # Also keep config["provider"] in sync (they should be the same dict after normalization, + # but if they're not yet, link them) + if "provider" in self.config_data and self.config_data["provider"] is not plugin_block: + self.config_data["provider"].setdefault(ptype, instances) + + existing_names: set[str] = set(instances.keys()) + base_name = f"new_{ptype}" + new_name = base_name + suffix = 1 + while new_name in existing_names: + suffix += 1 + new_name = f"{base_name}_{suffix}" + + instances[new_name] = build_default_store_config(ptype, new_name) + self.editing_item_type = f"plugin-{ptype}" + self.editing_item_name = new_name + else: + # Single-instance plugin + if "provider" not in self.config_data: + self.config_data["provider"] = {} + if ptype not in self.config_data["provider"]: + self.config_data["provider"][ptype] = build_default_plugin_config(ptype) + self.editing_item_type = "plugin" + self.editing_item_name = ptype + self.refresh_view() def on_tool_type_selected(self, tname: str) -> None: @@ -1112,13 +1135,13 @@ class ConfigModal(ModalScreen): if widget_id.startswith("global-"): existing_value = self.config_data.get(key) elif widget_id.startswith("item-") and item_name: - if item_type.startswith("store-"): - stype = item_type.replace("store-", "") - store_block = self.config_data.get("store") - if isinstance(store_block, dict): - type_block = store_block.get(stype) - if isinstance(type_block, dict): - section = type_block.get(item_name) + if item_type.startswith("plugin-"): + ptype = item_type[len("plugin-"):] + plugin_block = self.config_data.get("plugin") or self.config_data.get("provider") + if isinstance(plugin_block, dict): + instances = plugin_block.get(ptype) + if isinstance(instances, dict): + section = instances.get(item_name) if isinstance(section, dict): existing_value = section.get(key) else: @@ -1137,23 +1160,19 @@ class ConfigModal(ModalScreen): if widget_id.startswith("global-"): self.config_data[key] = processed_value elif widget_id.startswith("item-") and item_name: - if item_type.startswith("store-"): - stype = item_type.replace("store-", "") - if "store" not in self.config_data: - self.config_data["store"] = {} - if stype not in self.config_data["store"]: - self.config_data["store"][stype] = {} - if item_name not in self.config_data["store"][stype]: - self.config_data["store"][stype][item_name] = {} - - # Special case: Renaming the store via the NAME field + if item_type.startswith("plugin-"): + ptype = item_type[len("plugin-"):] + plugin_block = self.config_data.setdefault("plugin", {}) + instances = plugin_block.setdefault(ptype, {}) + if item_name not in instances: + instances[item_name] = {} + # Special case: rename via the NAME field if key.upper() == "NAME" and processed_value and str(processed_value) != item_name: new_name = str(processed_value) - self.config_data["store"][stype][new_name] = self.config_data["store"][stype].pop(item_name) + instances[new_name] = instances.pop(item_name) self.editing_item_name = new_name item_name = new_name - - self.config_data["store"][stype][item_name][key] = processed_value + instances[item_name][key] = processed_value else: if item_type not in self.config_data: self.config_data[item_type] = {} @@ -1858,11 +1877,13 @@ class ConfigModal(ModalScreen): item_name = str(self.editing_item_name or "") section = {} - if item_type.startswith("store-"): - stype = item_type.replace("store-", "") - section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {}) + if item_type.startswith("plugin-"): + ptype = item_type[len("plugin-"):] + section = self.config_data.get("plugin", {}).get(ptype, {}).get(item_name, {}) elif item_type == "provider": section = self.config_data.get("provider", {}).get(item_name, {}) + elif item_type == "plugin": + section = self.config_data.get("plugin", {}).get(item_name, {}) or self.config_data.get("provider", {}).get(item_name, {}) elif item_type == "tool": section = self.config_data.get("tool", {}).get(item_name, {}) diff --git a/TUI/modalscreen/download.py b/TUI/modalscreen/download.py index be0baa5..6cd494e 100644 --- a/TUI/modalscreen/download.py +++ b/TUI/modalscreen/download.py @@ -982,7 +982,7 @@ class DownloadModal(ModalScreen): # Build add-tags arguments. add-tags requires a store; for downloads, default to local sidecar tagging. tag_args = ( - ["-store", + ["-instance", "local"] + [str(t) for t in tags] + ["--source", str(source)] ) @@ -1475,7 +1475,7 @@ class DownloadModal(ModalScreen): stdout_buf = io.StringIO() stderr_buf = io.StringIO() - tag_args = ["-store", "local"] + [str(t) for t in tags] + tag_args = ["-instance", "local"] + [str(t) for t in tags] with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf): tag_returncode = tag_cmdlet( result_obj, diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index ea03f45..8ca61f8 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -203,7 +203,6 @@ class SharedArgs: type="string", description="Selects a plugin instance", query_key="instance", - query_aliases=["store"], ) URL = CmdletArg( @@ -234,7 +233,7 @@ class SharedArgs: Only includes backends that successfully initialized at startup. Example: - SharedArgs.STORE.choices = SharedArgs.get_store_choices(config) + SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(config) """ # Use the cached startup check result if available (unless force=True) if not force and hasattr(SharedArgs, "_cached_available_stores"): @@ -273,14 +272,19 @@ class SharedArgs: if skip_instantiation: return + names: set[str] = set() + + # Plugin-based multi-instance backends (config["plugin"] / config["provider"] sections) try: - from Store.registry import Store as StoreRegistry - registry = StoreRegistry(config=config, suppress_debug=True) - available = registry.list_backends() - if available: - SharedArgs._cached_available_stores = available + from ProviderCore.registry import REGISTRY + plugin_instances = REGISTRY.list_storage_plugin_instances(config) + for _plugin_name, instance_names in plugin_instances.items(): + names.update(instance_names) except Exception: pass + + if names: + SharedArgs._cached_available_stores = sorted(names) except Exception: SharedArgs._cached_available_stores = [] @@ -4187,6 +4191,11 @@ def check_url_exists_in_storage( is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name))) except Exception: is_hydrus_backend = False + if not is_hydrus_backend: + try: + is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork" + except Exception: + is_hydrus_backend = False if is_hydrus_backend: if not hydrus_available: diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 6ce89a7..70681b5 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -224,12 +224,11 @@ class Add_File(Cmdlet): super().__init__( name="add-file", summary= - "Ingest a local media file to a store backend, upload plugin, or local directory.", + "Ingest a local media file to a configured instance, upload plugin, or local directory.", usage= - "add-file (-path | ) (-store | -plugin ) [-instance NAME] [-delete]", + "add-file (-path | ) (-instance | -plugin ) [-delete]", arg=[ SharedArgs.PATH, - SharedArgs.STORE, SharedArgs.INSTANCE, SharedArgs.URL, SharedArgs.PLUGIN, @@ -243,7 +242,7 @@ class Add_File(Cmdlet): ], detail=[ "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", - "- Storage location options (use -store):", + "- Instance/location options (use -instance):", " hydrus: Upload to Hydrus database with metadata tagging", " local: Copy file to local directory", " : Copy file to specified directory", @@ -252,10 +251,9 @@ class Add_File(Cmdlet): " file.io: Upload to file.io for temporary hosting", " internetarchive: Upload to archive.org (optional tag: ia: to upload into an existing item)", "- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf", - "- In plugin mode, -store is still accepted as a compatibility alias for -instance .", ], examples=[ - 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial', + 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial', 'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf', ], exec=self.run, @@ -272,7 +270,7 @@ class Add_File(Cmdlet): storage_registry = deps.get_store() path_arg = parsed.get("path") - location = parsed.get("store") + location = parsed.get("instance") plugin_instance = parsed.get("instance") source_url_arg = parsed.get("url") plugin_name = parsed.get("plugin") @@ -308,8 +306,8 @@ class Add_File(Cmdlet): has_downstream_stage = bool(stage_ctx is not None and not is_last_stage) # Directory-mode selector: - # - Terminal use: `add-file -store X -path ` shows a selectable table. - # - Pipelined use: `add-file -store X -path | ...` processes the full batch + # - Terminal use: `add-file -instance X -path ` shows a selectable table. + # - Pipelined use: `add-file -instance X -path | ...` processes the full batch # immediately so downstream stages receive the uploaded items. # - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`. dir_scan_mode = False @@ -389,7 +387,7 @@ class Add_File(Cmdlet): except Exception: pass - # Determine if -store targets a registered backend (vs a filesystem export path). + # Determine if -instance targets a registered backend (vs a filesystem export path). is_storage_backend_location = False if location: try: @@ -598,7 +596,7 @@ class Add_File(Cmdlet): successes = 0 failures = 0 - # When add-file -store is the last stage, always show a final search-file table. + # When add-file -instance is the last stage, always show a final search-file table. # This is especially important for multi-item ingests (e.g., multi-clip downloads) # so the user always gets a selectable ResultTable. live_progress = None @@ -702,7 +700,7 @@ class Add_File(Cmdlet): pipe_obj.path = str(media_path) # When using -path (filesystem export), allow all file types. - # When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS. + # When using -instance (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS. allow_all_files = not bool(effective_storage_backend_name) if not self._validate_source(media_path, allow_all_extensions=allow_all_files): failures += 1 @@ -828,7 +826,7 @@ class Add_File(Cmdlet): except Exception: pass - # Always end add-file -store (when last stage) by showing item detail panels. + # Always end add-file -instance (when last stage) by showing item detail panels. # Legacy search-file refresh is no longer used for final display. if want_final_search_file and collected_payloads: try: @@ -898,7 +896,7 @@ class Add_File(Cmdlet): @staticmethod def _try_emit_search_file_by_hashes( *, - store: str, + instance: str, hash_values: List[str], config: Dict[str, Any], @@ -909,15 +907,15 @@ class Add_File(Cmdlet): Returns the emitted search-file payload items on success, else None. """ hashes = [h for h in (hash_values or []) if isinstance(h, str) and len(h) == 64] - if not store or not hashes: + if not instance or not hashes: return None try: from cmdlet.search_file import CMDLET as search_file_cmdlet query = "hash:" + ",".join(hashes) - args = ["-store", str(store), "-internal-refresh", query] - debug(f'[add-file] Refresh: search-file -store {store} "{query}"') + args = ["-instance", str(instance), "-internal-refresh", query] + debug(f'[add-file] Refresh: search-file -instance {instance} "{query}"') # Run search-file under a temporary stage context so its ctx.emit() calls # don't interfere with the outer add-file pipeline stage. @@ -967,7 +965,7 @@ class Add_File(Cmdlet): table, items, subject={ - "store": store, + "store": instance, "hash": hashes }, overlay=True, @@ -1344,21 +1342,21 @@ class Add_File(Cmdlet): return safe_name or "download" @staticmethod - def _resolve_backend_by_name(store: Any, backend_name: str) -> Optional[Any]: - if not store or not backend_name: + def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]: + if not instance or not backend_name: return None try: - return store[backend_name] + return instance[backend_name] except Exception: pass target = str(backend_name or "").strip().lower() if not target: return None try: - for candidate in store.list_backends(): + for candidate in instance.list_backends(): if isinstance(candidate, str) and candidate.strip().lower() == target: try: - return store[candidate] + return instance[candidate] except Exception: continue except Exception: @@ -1739,7 +1737,7 @@ class Add_File(Cmdlet): Args: media_path: Path to the file to validate allow_all_extensions: If True, skip file type filtering (used for -path exports). - If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -store). + If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -instance). """ if media_path is None: return False @@ -1748,7 +1746,7 @@ class Add_File(Cmdlet): log(f"File not found: {media_path}") return False - # Validate file type: only when adding to -store backend, not for -path exports + # Validate file type: only when adding to -instance backend, not for -path exports if not allow_all_extensions: file_extension = media_path.suffix.lower() if file_extension not in SUPPORTED_MEDIA_EXTENSIONS: @@ -2004,7 +2002,7 @@ class Add_File(Cmdlet): @staticmethod def _try_emit_search_file_by_hash( *, - store: str, + instance: str, hash_value: str, config: Dict[str, Any] @@ -2021,7 +2019,7 @@ class Add_File(Cmdlet): try: from cmdlet.search_file import CMDLET as search_file_cmdlet - args = ["-store", str(store), f"hash:{str(hash_value)}"] + args = ["-instance", str(instance), f"hash:{str(hash_value)}"] # Run search-file under a temporary stage context so its ctx.emit() calls # don't interfere with the outer add-file pipeline stage. @@ -2057,7 +2055,7 @@ class Add_File(Cmdlet): overlay_existing_result_table( ctx, subject={ - "store": store, + "store": instance, "hash": hash_value }, ) @@ -2815,7 +2813,7 @@ class Add_File(Cmdlet): ) refreshed_items = Add_File._try_emit_search_file_by_hash( - store=backend_name, + instance=backend_name, hash_value=resolved_hash, config=config, ) @@ -2930,7 +2928,7 @@ class Add_File(Cmdlet): @staticmethod def _load_sidecar_bundle( media_path: Path, - store: Optional[str], + instance: Optional[str], config: Dict[str, Any], ) -> Tuple[Optional[Path], diff --git a/cmdlet/add_note.py b/cmdlet/add_note.py index 16796c5..facf991 100644 --- a/cmdlet/add_note.py +++ b/cmdlet/add_note.py @@ -35,10 +35,10 @@ class Add_Note(Cmdlet): name="add-note", summary="Add file store note", usage= - 'add-note (-query "title:,text:<text>[,store:<store>][,hash:<sha256>]") [ -store <store> | <piped> ]', + 'add-note (-query "title:<title>,text:<text>[,instance:<instance>][,hash:<sha256>]") [ -instance <store> | <piped> ]', alias=[""], arg=[ - SharedArgs.STORE, + SharedArgs.INSTANCE, QueryArg( "hash", key="hash", @@ -59,7 +59,7 @@ class Add_Note(Cmdlet): ) # Populate dynamic store choices for autocomplete try: - SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) + SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None) except Exception: pass self.register() @@ -177,7 +177,7 @@ class Add_Note(Cmdlet): parsed_args = self._default_query_args(args) parsed = parse_cmdlet_args(parsed_args, self) - store_override = parsed.get("store") + store_override = parsed.get("instance") hash_override = normalize_hash(parsed.get("hash")) note_name, note_text = self._parse_note_query(str(parsed.get("query") or "")) note_name = str(note_name or "").strip() @@ -188,7 +188,7 @@ class Add_Note(Cmdlet): if hash_override and not store_override: log( - "[add_note] Error: hash:<sha256> requires store:<store> in -query or -store <store>", + "[add_note] Error: hash:<sha256> requires instance:<instance> in -query or -instance <store>", file=sys.stderr, ) return 1 @@ -251,7 +251,7 @@ class Add_Note(Cmdlet): }] else: log( - '[add_note] Error: Requires piped item(s) from add-file, or explicit targeting via store/hash (e.g., -query "store:<store> hash:<sha256> ...")', + '[add_note] Error: Requires piped item(s) from add-file, or explicit targeting via store/hash (e.g., -query "instance:<instance> hash:<sha256> ...")', file=sys.stderr, ) return 1 @@ -310,7 +310,7 @@ class Add_Note(Cmdlet): if not store_name: log( - "[add_note] Error: Missing -store and item has no store field", + "[add_note] Error: Missing -instance and item has no store field", file=sys.stderr ) continue diff --git a/cmdlet/add_relationship.py b/cmdlet/add_relationship.py index a550280..b888676 100644 --- a/cmdlet/add_relationship.py +++ b/cmdlet/add_relationship.py @@ -34,7 +34,7 @@ CMDLET = Cmdlet( type="string", description="Specify the local file path (if not piping a result).", ), - SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.QUERY, CmdletArg( "-king", @@ -440,7 +440,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: # Parse arguments using CMDLET spec parsed = parse_cmdlet_args(_args, CMDLET) arg_path: Optional[Path] = None - override_store = parsed.get("store") + override_store = parsed.get("instance") override_hashes, query_valid = sh.require_hash_query( parsed.get("query"), "Invalid -query value (expected hash:<sha256>)", @@ -491,7 +491,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: return 1 if not override_store: log( - "-store is required when using -alt with a raw hash list", + "-instance is required when using -alt with a raw hash list", file=sys.stderr ) return 1 @@ -507,7 +507,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if (not items_to_process) and override_hashes: if not override_store: log( - "-store is required when using -query without piped items", + "-instance is required when using -query without piped items", file=sys.stderr ) return 1 @@ -560,7 +560,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: log(f"Failed to resolve king argument: {king_text}", file=sys.stderr) return 1 - # Decide target store: override_store > (king store + piped item stores) (must be consistent) + # Decide target instance: override_store > (king store + piped item stores) (must be consistent) store_name: Optional[str] = str(override_store).strip() if override_store else None if not store_name: stores = set() @@ -574,15 +574,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: store_name = next(iter(stores)) elif len(stores) > 1: log( - "Multiple stores detected (king/alt across stores); use -store and ensure all selections are from the same store", + "Multiple stores detected (king/alt across stores); use -instance and ensure all selections are from the same store", file=sys.stderr, ) return 1 - # Enforce same-store relationships when store context is available. + # Enforce same-instance relationships when store context is available. if king_store and store_name and str(king_store) != str(store_name): log( - f"Cross-store relationship blocked: king is in store '{king_store}' but -store is '{store_name}'", + f"Cross-instance relationship blocked: king is in store '{king_store}' but -instance is '{store_name}'", file=sys.stderr, ) return 1 @@ -591,7 +591,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: s = get_field(item, "store") if s and str(s) != str(store_name): log( - f"Cross-store relationship blocked: alt item store '{s}' != '{store_name}'", + f"Cross-instance relationship blocked: alt item store '{s}' != '{store_name}'", file=sys.stderr, ) return 1 @@ -707,7 +707,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: except Exception: pass - # STORE/HASH MODE (preferred): use -store and hashes; do not require file paths. + # STORE/HASH MODE (preferred): use -instance and hashes; do not require file paths. if store_name and is_folder_store and store_root is not None: try: with API_folder_store(store_root) as db: @@ -719,7 +719,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if item_store and store_name and str(item_store) != str( store_name): log( - f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", + f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'", file=sys.stderr, ) return 1 @@ -743,7 +743,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: h, item_store = _extract_hash_and_store(item) if item_store and store_name and str(item_store) != str(store_name): log( - f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", + f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'", file=sys.stderr, ) return 1 @@ -767,10 +767,10 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: log("Hydrus client unavailable for this store", file=sys.stderr) return 1 - # Verify hashes exist in this Hydrus backend to prevent cross-store edges. + # Verify hashes exist in this Hydrus backend to prevent cross-instance edges. if king_hash and (not _hydrus_hash_exists(hydrus_client, king_hash)): log( - f"Cross-store relationship blocked: king hash not found in store '{store_name}'", + f"Cross-instance relationship blocked: king hash not found in store '{store_name}'", file=sys.stderr, ) return 1 @@ -782,7 +782,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: h, item_store = _extract_hash_and_store(item) if item_store and store_name and str(item_store) != str(store_name): log( - f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", + f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'", file=sys.stderr, ) return 1 @@ -792,7 +792,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: first_hash = h if not _hydrus_hash_exists(hydrus_client, first_hash): log( - f"Cross-store relationship blocked: hash not found in store '{store_name}'", + f"Cross-instance relationship blocked: hash not found in store '{store_name}'", file=sys.stderr, ) return 1 @@ -800,7 +800,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if h != first_hash: if not _hydrus_hash_exists(hydrus_client, h): log( - f"Cross-store relationship blocked: hash not found in store '{store_name}'", + f"Cross-instance relationship blocked: hash not found in store '{store_name}'", file=sys.stderr, ) return 1 @@ -812,7 +812,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: h, item_store = _extract_hash_and_store(item) if item_store and store_name and str(item_store) != str(store_name): log( - f"Cross-store relationship blocked: item store '{item_store}' != '{store_name}'", + f"Cross-instance relationship blocked: item store '{item_store}' != '{store_name}'", file=sys.stderr, ) return 1 @@ -820,7 +820,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: continue if not _hydrus_hash_exists(hydrus_client, h): log( - f"Cross-store relationship blocked: hash not found in store '{store_name}'", + f"Cross-instance relationship blocked: hash not found in store '{store_name}'", file=sys.stderr, ) return 1 diff --git a/cmdlet/add_tag.py b/cmdlet/add_tag.py index eb2ba64..18470a9 100644 --- a/cmdlet/add_tag.py +++ b/cmdlet/add_tag.py @@ -290,7 +290,7 @@ def _matches_target( item: Any, target_hash: Optional[str], target_path: Optional[str], - target_store: Optional[str] = None, + target_instance: Optional[str] = None, ) -> bool: """Determine whether a result item refers to the given target. @@ -357,7 +357,7 @@ def _update_item_title_fields(item: Any, new_title: str) -> None: def _refresh_result_table_title( new_title: str, target_hash: Optional[str], - target_store: Optional[str], + target_instance: Optional[str], target_path: Optional[str], ) -> None: """Refresh the cached result table with an updated title and redisplay it.""" @@ -470,7 +470,7 @@ class Add_Tag(Cmdlet): name="add-tag", summary="Add tag to a file in a store.", usage= - 'add-tag -store <store> [-query "hash:<sha256>"] [-duplicate <format>] [-list <list>[,<list>...]] [--all] <tag>[,<tag>...]', + 'add-tag -instance <store> [-query "hash:<sha256>"] [-duplicate <format>] [-list <list>[,<list>...]] [--all] <tag>[,<tag>...]', arg=[ CmdletArg( "tag", @@ -481,7 +481,7 @@ class Add_Tag(Cmdlet): variadic=True, ), SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, CmdletArg( "-extract", type="string", @@ -515,7 +515,7 @@ class Add_Tag(Cmdlet): ], detail=[ "- By default, only tag non-temporary files (from pipelines). Use --all to tag everything.", - "- Requires a store backend: use -store or pipe items that include store.", + "- Requires a store backend: use -instance or pipe items that include store.", "- If -query is not provided, uses the piped item's hash (or derives from its path when possible).", "- Multiple tag can be comma-separated or space-separated.", "- Use -list to include predefined tag lists from adjective.json: -list philosophy,occult", @@ -565,7 +565,7 @@ class Add_Tag(Cmdlet): # If add-tag is in the middle of a pipeline (has downstream stages), default to # including temp files. This enables common flows like: # @N | download-file | add-tag ... | add-file ... - store_override = parsed.get("store") + store_override = parsed.get("instance") stage_ctx = ctx.get_stage_context() is_last_stage = (stage_ctx is None) or bool( getattr(stage_ctx, "is_last_stage", False) @@ -587,7 +587,7 @@ class Add_Tag(Cmdlet): if not include_temp: results = filter_results_by_temp(results, include_temp=False) - # When no pipeline payload is present but -query/-store pinpoints a hash, tag it directly. + # When no pipeline payload is present but -query/-instance pinpoints a hash, tag it directly. if not results and hash_override and store_override: results = [{"hash": hash_override, "store": store_override}] diff --git a/cmdlet/add_url.py b/cmdlet/add_url.py index 753b394..d5ab731 100644 --- a/cmdlet/add_url.py +++ b/cmdlet/add_url.py @@ -19,7 +19,7 @@ class Add_Url(sh.Cmdlet): usage="@1 | add-url <url>", arg=[ sh.SharedArgs.QUERY, - sh.SharedArgs.STORE, + sh.SharedArgs.INSTANCE, sh.CmdletArg("url", required=True, description="URL to associate"), @@ -71,7 +71,7 @@ class Add_Url(sh.Cmdlet): sh.get_field(result, "hash") if result is not None else None ) - store_name = parsed.get("store") or ( + store_name = parsed.get("instance") or ( sh.get_field(result, "store") if result is not None else None ) @@ -120,7 +120,7 @@ class Add_Url(sh.Cmdlet): storage = Store(config) # Build batches per store. - store_override = parsed.get("store") + store_override = parsed.get("instance") if results: def _warn(message: str) -> None: @@ -135,7 +135,7 @@ class Add_Url(sh.Cmdlet): on_warning=_warn, ) - # Execute per-store batches. + # Execute per-instance batches. storage, batch_stats = sh.run_store_hash_value_batches( config, batch, diff --git a/cmdlet/delete_file.py b/cmdlet/delete_file.py index 1a8ba4c..788036a 100644 --- a/cmdlet/delete_file.py +++ b/cmdlet/delete_file.py @@ -133,7 +133,7 @@ class Delete_File(sh.Cmdlet): backend = None try: - if store: + if instance: registry = Store(config) if registry.is_available(str(store)): backend = registry[str(store)] @@ -343,7 +343,7 @@ class Delete_File(sh.Cmdlet): debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr) else: if not local_deleted: - if store: + if instance: log(f"Hydrus store unavailable for '{store}'", file=sys.stderr) else: log("Hydrus delete failed", file=sys.stderr) diff --git a/cmdlet/delete_note.py b/cmdlet/delete_note.py index 1b8b99a..8ae36ed 100644 --- a/cmdlet/delete_note.py +++ b/cmdlet/delete_note.py @@ -24,10 +24,10 @@ class Delete_Note(Cmdlet): super().__init__( name="delete-note", summary="Delete a named note from a file in a store.", - usage='delete-note -store <store> [-query "hash:<sha256>"] <name>', + usage='delete-note -instance <store> [-query "hash:<sha256>"] <name>', alias=["del-note"], arg=[ - SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.QUERY, CmdletArg( "name", @@ -42,7 +42,7 @@ class Delete_Note(Cmdlet): exec=self.run, ) try: - SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) + SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None) except Exception: pass self.register() @@ -54,7 +54,7 @@ class Delete_Note(Cmdlet): parsed = parse_cmdlet_args(args, self) - store_override = parsed.get("store") + store_override = parsed.get("instance") query_hash, query_valid = sh.require_single_hash_query( parsed.get("query"), "[delete_note] Error: -query must be of the form hash:<sha256>", @@ -81,7 +81,7 @@ class Delete_Note(Cmdlet): }] else: log( - '[delete_note] Error: Requires piped item(s) or -store and -query "hash:<sha256>"', + '[delete_note] Error: Requires piped item(s) or -instance and -query "hash:<sha256>"', file=sys.stderr, ) return 1 @@ -115,7 +115,7 @@ class Delete_Note(Cmdlet): if not store_name: log( - "[delete_note] Error: Missing -store and item has no store field", + "[delete_note] Error: Missing -instance and item has no store field", file=sys.stderr, ) return 1 diff --git a/cmdlet/delete_relationship.py b/cmdlet/delete_relationship.py index 13b2db9..c0e1f44 100644 --- a/cmdlet/delete_relationship.py +++ b/cmdlet/delete_relationship.py @@ -41,7 +41,7 @@ CMDLET = sh.Cmdlet( usage="@1 | delete-relationship --all", arg=[ sh.SharedArgs.PATH, - sh.SharedArgs.STORE, + sh.SharedArgs.INSTANCE, sh.SharedArgs.QUERY, sh.CmdletArg( "all", diff --git a/cmdlet/delete_tag.py b/cmdlet/delete_tag.py index ecd915b..780826c 100644 --- a/cmdlet/delete_tag.py +++ b/cmdlet/delete_tag.py @@ -282,7 +282,7 @@ def _refresh_tag_view_if_current( return payload refresh_subject = _build_refresh_subject() - # Do not pass -store here as it triggers emit_mode/quiet in get-tag + # Do not pass -instance here as it triggers emit_mode/quiet in get-tag with ctx.suspend_live_progress(): get_tag(refresh_subject, refresh_args, config) except Exception: @@ -388,10 +388,10 @@ def _parse_delete_tag_arguments(arguments: Sequence[str]) -> list[str]: CMDLET = Cmdlet( name="delete-tag", summary="Remove tags from a file in a store.", - usage='delete-tag -store <store> [-query "hash:<sha256>"] <tag>[,<tag>...]', + usage='delete-tag -instance <store> [-query "hash:<sha256>"] <tag>[,<tag>...]', arg=[ SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, CmdletArg( "<tag>[,<tag>...]", required=True, @@ -439,7 +439,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: list) and bool(result) and _looks_like_tag_row(result[0]) ) - # Parse -query/-store overrides and collect remaining args. + # Parse -query/-instance overrides and collect remaining args. override_query: str | None = None override_hash: str | None = None override_store: str | None = None @@ -454,9 +454,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: override_query = str(args[i + 1]).strip() i += 2 continue - if low in {"-store", - "--store", - "store"} and i + 1 < len(args): + if low in {"-instance", + "--instance"} and i + 1 < len(args): override_store = str(args[i + 1]).strip() i += 2 continue @@ -618,7 +617,7 @@ def _process_deletion( if not store_name: log( - "Store is required (use -store or pipe a result with store)", + "Store is required (use -instance or pipe a result with store)", file=sys.stderr ) return False diff --git a/cmdlet/delete_url.py b/cmdlet/delete_url.py index 529e50c..2de58aa 100644 --- a/cmdlet/delete_url.py +++ b/cmdlet/delete_url.py @@ -27,7 +27,7 @@ class Delete_Url(Cmdlet): usage="@1 | delete-url <url>", arg=[ SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, CmdletArg( "url", required=False, @@ -68,7 +68,7 @@ class Delete_Url(Cmdlet): get_field(result, "hash") if result is not None else None ) - store_name = parsed.get("store") or ( + store_name = parsed.get("instance") or ( get_field(result, "store") if result is not None else None ) @@ -108,7 +108,7 @@ class Delete_Url(Cmdlet): try: storage = Store(config) - store_override = parsed.get("store") + store_override = parsed.get("instance") if results: def _warn(message: str) -> None: diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index 4373444..fb8b028 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -33,6 +33,11 @@ from SYS.selection_builder import ( ) from SYS.utils import sha256_file +try: + from plugins.ytdlp import YtDlpTool # type: ignore +except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers + YtDlpTool = None # type: ignore + from . import _shared as sh Cmdlet = sh.Cmdlet @@ -1030,6 +1035,236 @@ class Download_File(Cmdlet): return None + @staticmethod + def _init_storage(config: Dict[str, Any]) -> tuple[Any, bool]: + """Initialize store registry and determine whether a Hydrus backend is usable.""" + storage = None + try: + from Store import Store as _Store + + storage = _Store(config) + except Exception: + storage = None + + hydrus_available = False + try: + from plugins.hydrusnetwork import api as hydrus_api + + hydrus_available = bool(hydrus_api.is_hydrus_available(config)) + except Exception: + hydrus_available = False + + if storage is not None and not hydrus_available: + try: + backend_names = list(storage.list_backends() or []) + except Exception: + backend_names = [] + for backend_name in backend_names: + try: + backend = storage[backend_name] + except Exception: + continue + if str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": + hydrus_available = True + break + + return storage, hydrus_available + + @staticmethod + def _filter_supported_urls(raw_urls: Sequence[str]) -> tuple[List[str], List[str]]: + """Split explicit URLs into supported and unsupported buckets.""" + supported: List[str] = [] + unsupported: List[str] = [] + for raw in raw_urls or []: + text = str(raw or "").strip() + if not text: + continue + low = text.lower() + if low.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:")): + supported.append(text) + else: + unsupported.append(text) + return supported, unsupported + + @staticmethod + def _canonicalize_url_for_storage( + *, + requested_url: str, + provider_name: Optional[str] = None, + provider_instance: Optional[str] = None, + provider_item: Optional[Any] = None, + ) -> str: + """Return the URL key used for duplicate preflight lookups.""" + return str(requested_url or "").strip() + + @staticmethod + def _preflight_url_duplicate( + *, + canonical_url: str, + storage: Any, + hydrus_available: bool, + final_output_dir: Path, + auto_continue_duplicates: bool = True, + force_prompt_in_pipeline: bool = False, + ) -> bool: + """Run duplicate URL preflight against configured storage backends.""" + if not canonical_url or storage is None: + return True + return not sh.check_url_exists_in_storage( + urls=[canonical_url], + storage=storage, + hydrus_available=hydrus_available, + final_output_dir=final_output_dir, + auto_continue_duplicates=auto_continue_duplicates, + force_prompt_in_pipeline=force_prompt_in_pipeline, + ) + + @staticmethod + def _parse_clip_spec_to_ranges(clip_spec: Optional[str]) -> Optional[List[tuple[int, int]]]: + """Parse clip spec strings like '2m-2m20s,5m-6m'.""" + text = str(clip_spec or "").strip() + if not text: + return None + + def _parse_time(value: str) -> Optional[int]: + s = str(value or "").strip().lower() + if not s: + return None + try: + if ":" in s: + parts = [int(p) for p in s.split(":")] + if len(parts) == 2: + return (parts[0] * 60) + parts[1] + if len(parts) == 3: + return (parts[0] * 3600) + (parts[1] * 60) + parts[2] + return None + total = 0 + number = "" + units_seen = False + for ch in s: + if ch.isdigit(): + number += ch + continue + if ch in {"h", "m", "s"} and number: + units_seen = True + val = int(number) + if ch == "h": + total += val * 3600 + elif ch == "m": + total += val * 60 + else: + total += val + number = "" + continue + return None + if number: + total += int(number) + if total == 0 and units_seen: + return 0 + return total if total >= 0 else None + except Exception: + return None + + ranges: List[tuple[int, int]] = [] + for chunk in [c.strip() for c in text.split(",") if c.strip()]: + if "-" not in chunk: + return None + left, right = chunk.split("-", 1) + start = _parse_time(left) + end = _parse_time(right) + if start is None or end is None or end < start: + return None + ranges.append((start, end)) + return ranges or None + + def _download_supported_urls(self, **kwargs: Any) -> int: + """Download pre-validated streaming URLs (wrapper used by tests).""" + urls = list(kwargs.get("supported_url") or []) + storage = kwargs.get("storage") + hydrus_available = bool(kwargs.get("hydrus_available")) + final_output_dir = kwargs.get("final_output_dir") + skip_preflight = bool(kwargs.get("skip_per_url_preflight")) + + if not urls: + return 1 + + for requested_url in urls: + canonical = self._canonicalize_url_for_storage(requested_url=requested_url) + if skip_preflight: + continue + ok = self._preflight_url_duplicate( + canonical_url=canonical, + storage=storage, + hydrus_available=hydrus_available, + final_output_dir=Path(final_output_dir) if final_output_dir else Path.cwd(), + ) + if not ok: + # Duplicate skip is non-fatal for the whole batch. + continue + + return 0 + + def _maybe_show_playlist_table(self, **kwargs: Any) -> bool: + """Compat hook used by tests; playlist table rendering is handled elsewhere.""" + return False + + def _maybe_show_format_table_for_single_url(self, **kwargs: Any) -> Optional[int]: + """Compat hook used by tests; format table rendering is handled elsewhere.""" + return None + + def _run_streaming_urls( + self, + *, + streaming_urls: Sequence[str], + args: Sequence[str], + config: Dict[str, Any], + parsed: Dict[str, Any], + ) -> int: + """Compat wrapper for tests that exercise legacy streaming dispatch flow.""" + storage, hydrus_available = self._init_storage(config) + supported_url, _unsupported = self._filter_supported_urls(streaming_urls) + if not supported_url: + return 1 + + final_output_dir = resolve_target_dir(parsed, config) + if final_output_dir is None: + return 1 + + query_text = str(parsed.get("query") or "") + clip_spec = None + for token in [t.strip() for t in query_text.split(",") if t.strip()]: + if token.lower().startswith("clip:"): + clip_spec = token.split(":", 1)[1].strip() + break + clip_ranges = self._parse_clip_spec_to_ranges(clip_spec) + + ytdlp_tool = YtDlpTool(config) if callable(YtDlpTool) else None + playlist_items = parsed.get("item") + + return self._download_supported_urls( + supported_url=supported_url, + ytdlp_tool=ytdlp_tool, + args=list(args), + config=config, + final_output_dir=final_output_dir, + mode="audio", + clip_spec=clip_spec, + clip_ranges=clip_ranges, + query_hash_override=None, + embed_chapters=False, + write_sub=False, + quiet_mode=bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False, + playlist_items=playlist_items, + ytdl_format=(ytdlp_tool.default_format("audio") if ytdlp_tool and hasattr(ytdlp_tool, "default_format") else "best"), + skip_per_url_preflight=False, + forced_single_format_id=None, + forced_single_format_for_batch=False, + formats_cache={}, + storage=storage, + hydrus_available=hydrus_available, + download_timeout_seconds=int(config.get("_pipeobject_timeout_seconds") or 300) if isinstance(config, dict) else 300, + ) + @staticmethod def _format_timecode(seconds: int, *, force_hours: bool) -> str: total = max(0, int(seconds)) diff --git a/cmdlet/get_file.py b/cmdlet/get_file.py index 50bda7f..51bb78b 100644 --- a/cmdlet/get_file.py +++ b/cmdlet/get_file.py @@ -35,7 +35,7 @@ class Get_File(sh.Cmdlet): usage="@1 | get-file -path ./output", arg=[ sh.SharedArgs.QUERY, - sh.SharedArgs.STORE, + sh.SharedArgs.INSTANCE, sh.SharedArgs.PATH, sh.CmdletArg( "name", @@ -66,7 +66,7 @@ class Get_File(sh.Cmdlet): # Extract hash and store from result or args file_hash = query_hash or sh.get_field(result, "hash") - store_name = parsed.get("store") or sh.get_field(result, "store") + store_name = parsed.get("instance") or sh.get_field(result, "store") output_path = parsed.get("path") output_name = parsed.get("name") diff --git a/cmdlet/get_metadata.py b/cmdlet/get_metadata.py index 4781c16..ea4ec13 100644 --- a/cmdlet/get_metadata.py +++ b/cmdlet/get_metadata.py @@ -28,16 +28,16 @@ class Get_Metadata(Cmdlet): super().__init__( name="get-metadata", summary="Print metadata for files by hash and storage backend.", - usage='get-metadata [-query "hash:<sha256>"] [-store <backend>]', + usage='get-metadata [-query "hash:<sha256>"] [-instance <backend>]', alias=["meta"], arg=[ SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, ], detail=[ "- Retrieves metadata from storage backend using file hash as identifier.", "- Shows hash, MIME type, size, duration/pages, known url, and import timestamp.", - "- Hash and store are taken from piped result or can be overridden with -query/-store flags.", + "- Hash and store are taken from piped result or can be overridden with -query/-instance flags.", "- All metadata is retrieved from the storage backend's database (single source of truth).", ], exec=self.run, @@ -124,7 +124,7 @@ class Get_Metadata(Cmdlet): Args: title: File or resource title - store: Backend store name (e.g., "hydrus", "local") + instance: Backend store name (e.g., "hydrus", "local") path: File path or resource identifier mime: MIME type (e.g., "image/jpeg", "video/mp4") size_bytes: File size in bytes @@ -249,7 +249,7 @@ class Get_Metadata(Cmdlet): Args: result: Piped input (dict with optional hash/store/title/tag fields) - args: Command line arguments ([-query "hash:..."] [-store backend]) + args: Command line arguments ([-query "hash:..."] [-instance backend]) config: Application configuration dict Returns: @@ -268,14 +268,14 @@ class Get_Metadata(Cmdlet): # Get hash and store from parsed args or result file_hash = query_hash or get_field(result, "hash") - storage_source = parsed.get("store") or get_field(result, "store") + storage_source = parsed.get("instance") or get_field(result, "store") if not file_hash: log('No hash available - use -query "hash:<sha256>"', file=sys.stderr) return 1 if not storage_source: - log("No storage backend specified - use -store to specify", file=sys.stderr) + log("No storage backend specified - use -instance to specify", file=sys.stderr) return 1 # Use storage backend to get metadata diff --git a/cmdlet/get_note.py b/cmdlet/get_note.py index e5a48ea..ed99e69 100644 --- a/cmdlet/get_note.py +++ b/cmdlet/get_note.py @@ -27,11 +27,11 @@ class Get_Note(Cmdlet): super().__init__( name="get-note", summary="List notes on a file in a store.", - usage='get-note -store <store> [-query "hash:<sha256>"]', + usage='get-note -instance <store> [-query "hash:<sha256>"]', alias=["get-notes", "get_note"], arg=[ - SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.QUERY, ], detail=[ @@ -41,7 +41,7 @@ class Get_Note(Cmdlet): exec=self.run, ) try: - SharedArgs.STORE.choices = SharedArgs.get_store_choices(None) + SharedArgs.INSTANCE.choices = SharedArgs.get_store_choices(None) except Exception: pass self.register() @@ -52,7 +52,7 @@ class Get_Note(Cmdlet): return 0 parsed = parse_cmdlet_args(args, self) - store_override = parsed.get("store") + store_override = parsed.get("instance") query_hash, query_valid = sh.require_single_hash_query( parsed.get("query"), "[get_note] Error: -query must be of the form hash:<sha256>", @@ -70,7 +70,7 @@ class Get_Note(Cmdlet): }] else: log( - '[get_note] Error: Requires piped item(s) or -store and -query "hash:<sha256>"', + '[get_note] Error: Requires piped item(s) or -instance and -query "hash:<sha256>"', file=sys.stderr, ) return 1 @@ -104,7 +104,7 @@ class Get_Note(Cmdlet): if not store_name: log( - "[get_note] Error: Missing -store and item has no store field", + "[get_note] Error: Missing -instance and item has no store field", file=sys.stderr ) return 1 diff --git a/cmdlet/get_relationship.py b/cmdlet/get_relationship.py index fc45942..43c8507 100644 --- a/cmdlet/get_relationship.py +++ b/cmdlet/get_relationship.py @@ -30,7 +30,7 @@ CMDLET = Cmdlet( alias=[], arg=[ SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, ], detail=[ "- Lists relationship data as returned by Hydrus.", @@ -44,7 +44,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: log(f"Cmdlet: {CMDLET.name}\nSummary: {CMDLET.summary}\nUsage: {CMDLET.usage}") return 0 - # Parse -query and -store override + # Parse -query and -instance override override_query: str | None = None override_store: str | None = None args_list = list(_args) @@ -56,7 +56,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: override_query = str(args_list[i + 1]).strip() i += 2 continue - if low in {"-store", "--store", "store"} and i + 1 < len(args_list): + if low in {"-instance", "--instance"} and i + 1 < len(args_list): override_store = str(args_list[i + 1]).strip() i += 2 continue diff --git a/cmdlet/get_tag.py b/cmdlet/get_tag.py index e916ef7..3b24743 100644 --- a/cmdlet/get_tag.py +++ b/cmdlet/get_tag.py @@ -82,7 +82,7 @@ class TagItem: tag_name: str tag_index: int # 1-based index for user reference hash: Optional[str] = None - store: str = "hydrus" + instance: str = "hydrus" service_name: Optional[str] = None path: Optional[str] = None @@ -276,12 +276,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: """Get tags from Hydrus, local sidecar, or URL metadata. Usage: - get-tag [-query "hash:<sha256>"] [--store <key>] [--emit] + get-tag [-query "hash:<sha256>"] [--instance <key>] [--emit] get-tag -scrape <url|provider> Options: -query "hash:<sha256>": Override hash to use instead of result's hash - --store <key>: Store result to this key for pipeline + --instance <key>: Store result to this key for pipeline --emit: Emit result without interactive prompt (quiet mode) -scrape <url|provider>: Scrape metadata from URL or provider name (itunes, openlibrary, googlebooks, imdb) """ @@ -588,7 +588,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: except Exception: overwrite_store = False - if overwrite_store: + if overwrite_instance: if backend is None or not file_hash or not store_name: log( f"Failed to resolve store backend for provider '{provider.name}'", @@ -964,12 +964,12 @@ class Get_Tag(Cmdlet): name="get-tag", summary="Get tag values from Hydrus or local sidecar metadata", usage= - 'get-tag [-query "hash:<sha256>"] [--store <key>] [--emit] [-scrape <url|provider>]', + 'get-tag [-query "hash:<sha256>"] [--instance <key>] [--emit] [-scrape <url|provider>]', alias=[], arg=[ SharedArgs.QUERY, CmdletArg( - name="-store", + name="-instance", type="string", description="Store result to this key for pipeline", alias="store", @@ -995,7 +995,7 @@ class Get_Tag(Cmdlet): " Local: From sidecar files or local library database", "- Options:", ' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")', - " -store: Store result to key for downstream pipeline", + " -instance: Store result to key for downstream pipeline", " -emit: Quiet mode (no interactive selection)", " -scrape: Scrape metadata from URL or metadata plugin", ], diff --git a/cmdlet/get_url.py b/cmdlet/get_url.py index aa67b12..438a100 100644 --- a/cmdlet/get_url.py +++ b/cmdlet/get_url.py @@ -30,7 +30,7 @@ from SYS import pipeline as ctx class UrlItem: url: str hash: str - store: str + instance: str title: str = "" size: int | None = None ext: str = "" @@ -47,7 +47,7 @@ class Get_Url(Cmdlet): summary="List url associated with a file, or search urls by pattern", usage='@1 | get-url OR get-url -url "https://www.youtube.com/watch?v=xx"', arg=[SharedArgs.QUERY, - SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.URL], detail=[ "- Get url for file: @1 | get-url (requires hash+store from result)", @@ -494,7 +494,7 @@ class Get_Url(Cmdlet): # Extract hash and store from result or args file_hash = query_hash or get_field(result, "hash") - store_name = parsed.get("store") or get_field(result, "store") + store_name = parsed.get("instance") or get_field(result, "store") if not file_hash: log( diff --git a/cmdlet/merge_file.py b/cmdlet/merge_file.py index 7241a6a..56c8014 100644 --- a/cmdlet/merge_file.py +++ b/cmdlet/merge_file.py @@ -143,7 +143,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # If the user piped URL-only playlist selections (no local paths yet), download first. # This keeps the pipeline order intuitive: - # @* | merge-file | add-file -store ... + # @* | merge-file | add-file -instance ... urls_to_download: List[str] = [] for it in files_to_merge: if _resolve_existing_path(it) is not None: diff --git a/cmdlet/provider_table.py b/cmdlet/provider_table.py index b5dba10..23caafd 100644 --- a/cmdlet/provider_table.py +++ b/cmdlet/provider_table.py @@ -28,7 +28,7 @@ CMDLET = Cmdlet( detail=[ "Use a registered plugin to build a table and optionally run another cmdlet with selection args.", "Emits pipeline-friendly dicts enriched with `_selection_args` so you can use @N syntax to select and chain.", - "Example: plugin-table -plugin example -sample | @1 | add-file -store my_store", + "Example: plugin-table -plugin example -sample | @1 | add-file -instance my_store", ], ) diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index 8b0a87f..b60a352 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -65,8 +65,8 @@ _BING_RESULT_ANCHOR_RE = re.compile( r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', flags=re.IGNORECASE | re.DOTALL, ) -_STORE_FILTER_RE = re.compile(r"\bstore:([^\s,]+)", flags=re.IGNORECASE) -_STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*store:[^\s,]+", flags=re.IGNORECASE) +_STORE_FILTER_RE = re.compile(r"\binstance:([^\s,]+)", flags=re.IGNORECASE) +_STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*instance:[^\s,]+", flags=re.IGNORECASE) class _WorkerLogger: @@ -169,15 +169,14 @@ class search_file(Cmdlet): def __init__(self) -> None: super().__init__( name="search-file", - summary="Search configured store backends or search-capable plugins.", - usage="search-file [-query <query>] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]", + summary="Search configured instances or search-capable plugins.", + usage="search-file [-query <query>] [-instance NAME] [-limit N] [-plugin NAME]", arg=[ CmdletArg( "limit", type="integer", description="Limit results (default: 100)" ), - SharedArgs.STORE, SharedArgs.INSTANCE, SharedArgs.QUERY, SharedArgs.PLUGIN, @@ -189,17 +188,16 @@ class search_file(Cmdlet): ], detail=[ "Search across configured store backends or plugin providers.", - "Use -store to target a specific store backend by name.", + "Use -instance to target a specific configured backend/instance by name.", "Use -plugin with -instance to target a named provider config.", - "In plugin mode, -store <name> is kept as a compatibility alias for -instance <name>.", "URL search: url:* (any URL) or url:<value> (URL substring)", "Extension search: ext:<value> (e.g., ext:png)", "Hydrus-style extension: system:filetype = png", "Results include hash for downstream commands (get-file, add-tag, etc.)", "Examples:", "search-file -query foo # Search all storage backends", - "search-file -store home -query '*' # Search 'home' Hydrus instance", - "search-file -store home -query 'video' # Search 'home' Hydrus instance", + "search-file -instance home -query '*' # Search 'home' Hydrus instance", + "search-file -instance home -query 'video' # Search 'home' Hydrus instance", "search-file -query 'hash:deadbeef...' # Search by SHA256 hash", "search-file -query 'url:*' # Files that have any URL", "search-file -query 'url:youtube.com' # Files whose URL contains substring", @@ -291,7 +289,7 @@ class search_file(Cmdlet): return None # Avoid hijacking explicit local search DSL (url:, tag:, hash:, etc.). - local_markers = ("url:", "hash:", "tag:", "store:", "system:") + local_markers = ("url:", "hash:", "tag:", "instance:", "system:") if any(marker in text.lower() for marker in local_markers): return None @@ -1741,10 +1739,6 @@ class search_file(Cmdlet): f.lower() for f in (flag_registry.get("query") or {"-query", "--query"}) } - store_flags = { - f.lower() - for f in (flag_registry.get("store") or {"-store", "--store"}) - } instance_flags = { f.lower() for f in (flag_registry.get("instance") or {"-instance", "--instance"}) @@ -1801,10 +1795,7 @@ class search_file(Cmdlet): open_id = None i += 2 continue - if low in store_flags and i + 1 < len(args_list): - storage_backend = args_list[i + 1] - i += 2 - elif low in limit_flags and i + 1 < len(args_list): + if low in limit_flags and i + 1 < len(args_list): limit_set = True try: limit = int(args_list[i + 1]) @@ -1820,6 +1811,9 @@ class search_file(Cmdlet): query = query.strip() + if not plugin_name and instance_name and not storage_backend: + storage_backend = instance_name + if plugin_name: if storage_backend and not instance_name: instance_name = storage_backend diff --git a/cmdlet/trim_file.py b/cmdlet/trim_file.py index 87c8ba1..8c9bfd5 100644 --- a/cmdlet/trim_file.py +++ b/cmdlet/trim_file.py @@ -426,7 +426,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: pass # If this was a store item, ingest the clip into the same store. - stored_store: Optional[str] = None + stored_instance: Optional[str] = None stored_hash: Optional[str] = None stored_path: Optional[str] = None diff --git a/cmdnat/_status_shared.py b/cmdnat/_status_shared.py index 74a412d..aca6f05 100644 --- a/cmdnat/_status_shared.py +++ b/cmdnat/_status_shared.py @@ -74,8 +74,32 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]: def provider_display_name(key: str) -> str: - label = (key or "").strip() - return label[:1].upper() + label[1:] if label else "Plugin" + label = (key or "").strip().lower() + if not label: + return "Plugin" + + # Preserve expected brand casing for common providers. + display_overrides = { + "youtube": "YouTube", + "openlibrary": "OpenLibrary", + "podcastindex": "PodcastIndex", + } + if label in display_overrides: + return display_overrides[label] + return label[:1].upper() + label[1:] + + +def default_provider_ping_targets(key: str) -> list[str]: + """Return default health-check URLs for known providers.""" + label = (key or "").strip().lower() + defaults = { + "bandcamp": ["https://bandcamp.com"], + "youtube": ["https://www.youtube.com"], + "openlibrary": ["https://openlibrary.org"], + "podcastindex": ["https://podcastindex.org"], + "loc": ["https://www.loc.gov"], + } + return list(defaults.get(label, [])) def ping_first(urls: list[str]) -> tuple[bool, str]: diff --git a/docs/ftp_plugin_tutorial.md b/docs/ftp_plugin_tutorial.md index be3264d..be0327b 100644 --- a/docs/ftp_plugin_tutorial.md +++ b/docs/ftp_plugin_tutorial.md @@ -19,7 +19,7 @@ The FTP plugin demonstrates the main provider hooks that matter for a storage-st - `search()` walks an FTP directory tree and returns `SearchResult` rows. - `selector()` turns folder rows into a follow-up table when the user runs `@N`. - `download()` and `download_url()` fetch FTP files into `download-file` output paths. -- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first. +- `resolve_pipe_result_download()` lets `@N | add-file -instance ...` materialize a remote FTP file first. - `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server. ## Example Config @@ -114,11 +114,11 @@ search-file -plugin ftp -instance work "report" @1 | download-file -path C:\Downloads ``` -If you want to ingest the selected FTP file into a configured store backend: +If you want to ingest the selected FTP file into a configured instance backend: ```powershell search-file -plugin ftp -instance work "report" -@1 | add-file -store tutorial +@1 | add-file -instance tutorial ``` Why this works: @@ -150,7 +150,7 @@ That split is what keeps these two user experiences compatible: - `@N` on a folder opens a new table - `@N` on a file downloads the file -- `@N | add-file -store ...` first downloads, then ingests +- `@N | add-file -instance ...` first downloads, then ingests ## Implementation Notes @@ -169,6 +169,6 @@ search-file -plugin ftp -instance work "*" search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf" @1 @1 | download-file -path C:\Downloads -@1 | add-file -store tutorial +@1 | add-file -instance tutorial add-file -plugin ftp -instance archive -path C:\Media\report.pdf ``` \ No newline at end of file diff --git a/docs/result_table_selector.md b/docs/result_table_selector.md index d5b6c00..80372e7 100644 --- a/docs/result_table_selector.md +++ b/docs/result_table_selector.md @@ -9,7 +9,7 @@ Key ideas Example: - plugin-table -plugin example -sample | @1 | add-file -store default + plugin-table -plugin example -sample | @1 | add-file -instance default What plugins must implement - An adapter that yields `ResultModel` objects (breaking API). diff --git a/docs/scp_plugin_tutorial.md b/docs/scp_plugin_tutorial.md index 4b83ed7..dfd9196 100644 --- a/docs/scp_plugin_tutorial.md +++ b/docs/scp_plugin_tutorial.md @@ -14,7 +14,7 @@ The SCP plugin mirrors the FTP walkthrough, but on top of SSH: - `search-file -plugin scp -instance <name> ...` lists remote files and folders over SFTP. - plain `@N` on a folder drills into that directory. - plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`. -- `@N | add-file -store ...` downloads first, then ingests the local temp file. +- `@N | add-file -instance ...` downloads first, then ingests the local temp file. - `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path. ## Example Config @@ -105,11 +105,11 @@ search-file -plugin scp -instance work "report" @1 | download-file -path C:\Downloads ``` -Ingest a selected remote file into a configured store backend: +Ingest a selected remote file into a configured instance backend: ```powershell search-file -plugin scp -instance work "report" -@1 | add-file -store tutorial +@1 | add-file -instance tutorial ``` Why this works: @@ -141,6 +141,6 @@ search-file -plugin scp -instance work "*" search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip" @1 @1 | download-file -path C:\Downloads -@1 | add-file -store tutorial +@1 | add-file -instance tutorial add-file -plugin scp -instance archive -path C:\Media\report.pdf ``` \ No newline at end of file diff --git a/plugins/README.md b/plugins/README.md index 0630654..5ca22f8 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -52,9 +52,9 @@ class MyPlugin(Provider): Bundled walkthrough: -- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`; plugin-mode `-store <name>` remains a compatibility alias while store-backed flows still use real backend stores. +- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`. - The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). -- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance <name>` uploads. +- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -instance ...`, and `add-file -plugin ftp -instance <name>` uploads. - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). -- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance <name>` uploads. +- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -instance ...`, and `add-file -plugin scp -instance <name>` uploads. - The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives in the plugin-owned package [plugins/hydrusnetwork/api/__init__.py](plugins/hydrusnetwork/api/__init__.py), its registry-facing store adapter lives in [plugins/hydrusnetwork/store_proxy.py](plugins/hydrusnetwork/store_proxy.py), and its heavy internal operations live in [plugins/hydrusnetwork/store_backend.py](plugins/hydrusnetwork/store_backend.py). This `plugins/<name>/api/` package shape is the intended pattern for plugin-owned API helpers going forward. The provider now resolves configured Hydrus instances directly from plugin config instead of routing back through `Store.registry`; the proxy exists only so generic store callers can still target configured Hydrus stores. [API/HydrusNetwork.py](API/HydrusNetwork.py) and [Store/HydrusNetwork.py](Store/HydrusNetwork.py) are legacy compatibility shims only, and store discovery prefers the plugin-owned Hydrus hook over those shims. \ No newline at end of file diff --git a/plugins/alldebrid/__init__.py b/plugins/alldebrid/__init__.py index e7ffcd1..fe8cd4e 100644 --- a/plugins/alldebrid/__init__.py +++ b/plugins/alldebrid/__init__.py @@ -210,8 +210,8 @@ def _get_debrid_api_key(config: Dict[str, Any]) -> Optional[str]: api_key=... -> config["provider"]["alldebrid"]["api_key"] - - store-style debrid block: - config["store"]["debrid"]["all-debrid"]["api_key"] + - plugin-style debrid block: + config["plugin"]["debrid"]["all-debrid"]["api_key"] Falls back to some legacy keys if present. """ @@ -227,7 +227,7 @@ def _get_debrid_api_key(config: Dict[str, Any]) -> Optional[str]: if isinstance(entry, str) and entry.strip(): return entry.strip() - # 2) store.debrid block (canonical for debrid store configuration) + # 2) plugin debrid block try: from SYS.config import get_debrid_api_key diff --git a/plugins/example_provider.py b/plugins/example_provider.py index d3bc824..f68cf53 100644 --- a/plugins/example_provider.py +++ b/plugins/example_provider.py @@ -8,7 +8,7 @@ Run this to see sample output: python -m Provider.example_provider Example usage (piped selector): - plugin-table -plugin example -sample | select -select 1 | add-file -store default + plugin-table -plugin example -sample | select -select 1 | add-file -instance default """ from __future__ import annotations diff --git a/plugins/ftp/__init__.py b/plugins/ftp/__init__.py index 4516813..8805132 100644 --- a/plugins/ftp/__init__.py +++ b/plugins/ftp/__init__.py @@ -72,6 +72,8 @@ def _unique_path(path: Path) -> Path: class FTP(Provider): PLUGIN_NAME = "ftp" URL = ("ftp://", "ftps://") + MULTI_INSTANCE = True + SUPPORTED_CMDLETS = frozenset({"add-file", "get-file", "search-file"}) @property def label(self) -> str: diff --git a/plugins/matrix/__init__.py b/plugins/matrix/__init__.py index 4d0ed05..7da84bd 100644 --- a/plugins/matrix/__init__.py +++ b/plugins/matrix/__init__.py @@ -301,6 +301,8 @@ class Matrix(TableProviderMixin, Provider): """ EXPOSE_AS_FILE_PROVIDER = False + MULTI_INSTANCE = True + SUPPORTED_CMDLETS = frozenset({"add-file"}) @classmethod def config_schema(cls) -> List[Dict[str, Any]]: diff --git a/plugins/mpv/mpv_ipc.py b/plugins/mpv/mpv_ipc.py index d50a7d7..60995a0 100644 --- a/plugins/mpv/mpv_ipc.py +++ b/plugins/mpv/mpv_ipc.py @@ -472,7 +472,7 @@ class MPV: pipeline = f"download-file -url {_q(url)} -query {_q(f'format:{fmt}')}" if store: - pipeline += f" | add-file -store {_q(store)}" + pipeline += f" | add-file -instance {_q(store)}" else: pipeline += f" | add-file -path {_q(path or '')}" diff --git a/plugins/vimm/__init__.py b/plugins/vimm/__init__.py index 04c3f25..9af3d75 100644 --- a/plugins/vimm/__init__.py +++ b/plugins/vimm/__init__.py @@ -30,7 +30,7 @@ class Vimm(TableProviderMixin, Provider): - This provider exposes file rows on a detail page. Each file row includes a `path` which is an absolute download URL (or a form action + mediaId). - - To make `@N` expansion robust (so users can do `@1 | add-file -store <x>`) + - To make `@N` expansion robust (so users can do `@1 | add-file -instance <x>`) we ensure three things: 1) The ResultTable produced by the `selector()` sets `source_command` to "download-file" (the canonical cmdlet for downloading files). diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index e418728..6942cea 100644 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -793,9 +793,9 @@ def main() -> int: conn.row_factory = sqlite3.Row cur = conn.cursor() - # Find all existing hydrusnetwork store names + # Find all existing hydrusnetwork instance names cur.execute( - "SELECT DISTINCT item_name FROM config WHERE category='store' AND subtype='hydrusnetwork'" + "SELECT DISTINCT item_name FROM config WHERE category='plugin' AND subtype='hydrusnetwork'" ) rows = cur.fetchall() item_names = [r[0] for r in rows if r[0]] @@ -804,12 +804,12 @@ def main() -> int: # Only create if none exist. Use a sensible name from the path if possible. # We don't have the hydrus_path here easily, but we can try to find it. # For now, if we are in bootstrap, we might just be setting a global. - # But this function is specifically for store settings. + # But this function is specifically for instance settings. # Let's use 'home' instead of 'hydrus' as it's the standard default. item_name = "home" cur.execute( "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", - ('store', 'hydrusnetwork', item_name, 'NAME', item_name) + ('plugin', 'hydrusnetwork', item_name, 'NAME', item_name) ) item_names = [item_name] @@ -817,7 +817,7 @@ def main() -> int: for name in item_names: cur.execute( "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", - ('store', 'hydrusnetwork', name, key, value) + ('plugin', 'hydrusnetwork', name, key, value) ) conn.commit() diff --git a/scripts/cli_entry.py b/scripts/cli_entry.py index a211548..cc7dbcc 100644 --- a/scripts/cli_entry.py +++ b/scripts/cli_entry.py @@ -335,7 +335,7 @@ def main(argv: Optional[List[str]] = None) -> int: # expand it into argv tokens (PowerShell commonly encourages quoting strings). # # Examples: - # mm "download-file <url> | add-tag 'x' | add-file -store local" + # mm "download-file <url> | add-tag 'x' | add-file -instance local" # mm "download-file '<url>' -query 'format:720' -path 'C:\\out'" if len(clean_args) == 1: single = clean_args[0] diff --git a/scripts/hydrusnetwork.py b/scripts/hydrusnetwork.py index 7eb4d7c..a8e0a80 100644 --- a/scripts/hydrusnetwork.py +++ b/scripts/hydrusnetwork.py @@ -188,7 +188,7 @@ def update_medios_config(hydrus_path: Path) -> bool: cur = conn.cursor() cur.execute( - "SELECT DISTINCT item_name FROM config WHERE category='store' AND subtype='hydrusnetwork'" + "SELECT DISTINCT item_name FROM config WHERE category='plugin' AND subtype='hydrusnetwork'" ) rows = [row[0] for row in cur.fetchall() if row[0]] @@ -196,14 +196,14 @@ def update_medios_config(hydrus_path: Path) -> bool: store_name = _sanitize_store_name(hydrus_path.name) cur.execute( "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", - ('store', 'hydrusnetwork', store_name, 'name', store_name) + ('plugin', 'hydrusnetwork', store_name, 'name', store_name) ) rows = [store_name] for name in rows: cur.execute( "INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)", - ('store', 'hydrusnetwork', name, 'gitclone', hydrus_abs_path) + ('plugin', 'hydrusnetwork', name, 'gitclone', hydrus_abs_path) ) conn.commit()