update refactoring and add new features

.env.example

@@ -0,0 +1,115 @@
+# =============================================================================
+# Medeia-Macina Environment Variables
+# =============================================================================
+# Copy this file to .env and adjust values as needed.
+#
+# All variables are optional unless noted otherwise. The application auto-detects
+# sensible defaults for most settings.
+
+# ---------------------------------------------------------------------------
+# Core / Debug
+# ---------------------------------------------------------------------------
+
+# Enable debug mode (verbose logging, diagnostic output, rich tracebacks).
+# Set to "1" to enable.
+# Default: disabled (unless the database config key `debug` is "1"/"true"/"yes"/"on")
+MM_DEBUG=0
+
+# Enable raw HTTP request/response body logging in debug output. When set to
+# "0", "false", "no", or "off", raw bodies are suppressed even while MM_DEBUG
+# is active. Any other value (or unset) follows MM_DEBUG.
+MM_HTTP_RAW=0
+
+# ---------------------------------------------------------------------------
+# Repository / Paths
+# ---------------------------------------------------------------------------
+
+# Application root directory. Searched for medios.db, CLI.py, and config.conf.
+# Precedence: MM_REPO > MM_ROOT > REPO > auto-detection from CWD.
+MM_REPO=
+
+# Alternative root directory name (checked if MM_REPO is not set).
+MM_ROOT=
+
+# Legacy / generic repository path (checked after MM_REPO and MM_ROOT).
+REPO=
+
+# Semicolon-separated list of external plugin directories. Plugins placed in
+# these directories are discovered and registered at startup.
+# Example: C:\extra-plugins;C:\more-plugins
+MM_PLUGIN_PATH=
+
+# Legacy alias for MM_PLUGIN_PATH (checked second).
+MEDEIA_PLUGIN_PATH=
+
+# ---------------------------------------------------------------------------
+# Networking
+# ---------------------------------------------------------------------------
+
+# Custom SSL/TLS CA certificate bundle path (PEM format).
+# Overrides system-wide trust stores for all TLS connections made by the
+# application. Useful behind corporate MITM proxies or custom CAs.
+SSL_CERT_FILE=
+
+# ---------------------------------------------------------------------------
+# MPV Integration
+# ---------------------------------------------------------------------------
+
+# Override the default MPV IPC socket / named-pipe path. Used by the CLI,
+# the pipeline helper, and the lyrics overlay to communicate with mpv.
+# Default (Windows): \\.\pipe\mpv-medios-macina
+# Default (macOS/Linux): /tmp/mpv-medios-macina.sock
+MEDEIA_MPV_IPC=
+
+# Legacy alias for MEDEIA_MPV_IPC (checked second).
+MPV_IPC_SERVER=
+
+# Minimum log level for mpv's internal message stream forwarded to the helper
+# log. Valid values: "fatal", "error", "warn" (default), "info", "debug", "v",
+# "trace".
+MEDEIA_MPV_HELPER_MPVLOG=warn
+
+# ---------------------------------------------------------------------------
+# Dependencies / Updates
+# ---------------------------------------------------------------------------
+
+# Automatically update Python dependencies on client startup.
+# Set to "1" to force `pip install --upgrade` on the requirements file.
+MM_UPDATE_DEPS=0
+
+# ---------------------------------------------------------------------------
+# Playwright / Browser Automation
+# ---------------------------------------------------------------------------
+
+# Path to the ffmpeg executable. Used by Playwright for video recording and
+# processing. Resolution order:
+#   1. Plugin config key `plugin.playwright.ffmpeg_path`
+#   2. Environment variable `FFMPEG_PATH`
+#   3. Environment variable `PLAYWRIGHT_FFMPEG_PATH` (legacy)
+#   4. Project-bundled ffmpeg (MPV/ffmpeg/bin/)
+#   5. System PATH lookup (`which ffmpeg`)
+FFMPEG_PATH=
+
+# Legacy Playwright-specific ffmpeg path. Prefer `FFMPEG_PATH` instead.
+PLAYWRIGHT_FFMPEG_PATH=
+
+# ---------------------------------------------------------------------------
+# Internet Archive
+# ---------------------------------------------------------------------------
+
+# Internet Archive S3-like API access key. Used for uploads. Can also be
+# configured in the plugin section of the config (key: `access_key`).
+IA_ACCESS_KEY=
+
+# Internet Archive S3-like API secret key. Used for uploads. Can also be
+# configured in the plugin section of the config (key: `secret_key`).
+IA_SECRET_KEY=
+
+# ---------------------------------------------------------------------------
+# Soulseek
+# ---------------------------------------------------------------------------
+
+# Enable verbose aioslsk library output (normally suppressed).
+# Set to "1" to see debug-level noise from the Soulseek networking layer.
+DOWNLOAD_SOULSEEK_VERBOSE=0
+# Note: The old name `DOWNLOW_SOULSEEK_VERBOSE` (typo) is deprecated.

.gitignore

@@ -82,8 +82,6 @@ target/
 # IPython
 profile_default/
 ipython_config.py
-config.conf
-config.d/
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
@@ -162,13 +160,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
 # Ruff stuff:
 .ruff_cache/
 
@@ -205,7 +196,6 @@ luac.out
 
 # Shared objects (inc. Windows DLLs)
 *.dll
-*.so
 *.so.*
 *.dylib
 
@@ -218,12 +208,11 @@ luac.out
 *.hex
 
 
-config.conf
-config.d/
 MPV/ffmpeg/*
 
+config_backups/
+
 Log/
-Log/medeia_macina/telegram.session
 mpv_logs_with_db.txt
 *.session
 example.py
@@ -232,21 +221,19 @@ MPV/portable_config/watch_later*
 hydrusnetwork
 .style.yapf
 .yapfignore
-tests/
 scripts/mm.ps1
 scripts/mm
 ..style.yapf
 .yapfignore
 tmp_*
 *.secret
-authtoken.secret
 
 mypy.
 .idea
 medios.db
 medios*
 mypy.ini
-\logs\*
+logs/*
 logs*
 logs.db
 logs.db*

API/base.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from typing import Any, Dict, Optional
+import atexit
 import threading
 
 from .HTTP import HTTPClient
@@ -19,6 +20,7 @@ class API:
         self.timeout = float(timeout)
         self._http_client: Optional[HTTPClient] = None
         self._http_client_lock = threading.Lock()
+        atexit.register(self.close)
 
     def _get_http_client(self) -> HTTPClient:
         """Return a reusable opened HTTP client for this API instance."""
@@ -49,12 +51,6 @@ class API:
                 pass
             self._http_client = None
 
-    def __del__(self) -> None:
-        try:
-            self.close()
-        except Exception:
-            pass
-
     def _get_json(
         self,
         path: str,

API/cmdlet.py

@@ -69,7 +69,7 @@ def run_cmdlet(
 ) -> CmdletRunResult:
     """Run a single cmdlet programmatically and return structured results.
 
-    This is intended for TUI/webapp consumers that want cmdlet behavior without
+    This is intended for programmatic consumers that want cmdlet behavior without
     going through the interactive CLI loop.
 
     Notes:

CLI.py

@@ -827,9 +827,14 @@ class CmdletCompleter(Completer):
             return None
         raw_plugin = CmdletCompleter._flag_value(stage_tokens, "-plugin", "--plugin")
         if raw_plugin:
-            # Strip quotes if present, then normalize to lowercase
             stripped = CmdletCompleter._strip_quotes(str(raw_plugin or ""))
-            return stripped.strip().lower() if stripped else None
+            if not stripped:
+                return None
+            plugin_name = stripped.strip().lower()
+            if "," in plugin_name:
+                parts = [p.strip() for p in plugin_name.split(",") if p.strip()]
+                return parts[0] if parts else None
+            return plugin_name
         return None
 
     @staticmethod
@@ -1058,11 +1063,20 @@ class CmdletCompleter(Completer):
                 logical: idx for idx, logical in enumerate(file_stage_order)
             }
 
+        action_logicals = {logical for logical, _ in self._FILE_STAGE_ACTION_CMDLETS}
+        chosen_actions: Set[str] = set()
+        for token in stage_tokens:
+            logical = str(token).lstrip("-").strip().lower()
+            if logical in action_logicals:
+                chosen_actions.add(logical)
+
         filtered: List[Tuple[int, int, str]] = []
         for index, arg in enumerate(arg_names):
             logical = str(arg or "").lstrip("-").strip().lower()
             if file_stage_order is not None and logical not in file_stage_rank:
                 continue
+            if chosen_actions and logical in action_logicals and logical not in chosen_actions:
+                continue
             if logical == "open":
                 continue
             if logical == "instance":
@@ -3085,6 +3099,10 @@ Come to love it when others take what you share, as there is no greater joy
                     if "|" in tokens or (tokens and tokens[0].startswith("@")):
                         self._pipeline_executor.execute_tokens(tokens)
                     else:
+                        try:
+                            ctx.clear_pending_pipeline_tail()
+                        except Exception:
+                            pass
                         cmd_name = tokens[0].replace("_", "-").lower()
                         is_help = any(
                             arg in {"-help",

PluginCore/registry.py

@@ -330,36 +330,43 @@ class PluginRegistry:
         for plugin_dir in _iter_external_plugin_dirs():
             if self._is_builtin_package_dir(plugin_dir):
                 continue
+
+            plugin_dir_str = str(plugin_dir)
+            path_added = False
+            if plugin_dir_str and plugin_dir_str not in sys.path:
+                sys.path.insert(0, plugin_dir_str)
+                path_added = True
+
             try:
-                plugin_dir_str = str(plugin_dir)
+                for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir):
-                if plugin_dir_str and plugin_dir_str not in sys.path:
+                    if module_name in self._external_modules:
-                    sys.path.insert(0, plugin_dir_str)
+                        continue
-            except Exception:
-                pass
 
-            for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir):
+                    try:
-                if module_name in self._external_modules:
+                        if is_package:
-                    continue
+                            spec = importlib.util.spec_from_file_location(
+                                module_name,
+                                str(module_path),
+                                submodule_search_locations=[str(module_path.parent)],
+                            )
+                        else:
+                            spec = importlib.util.spec_from_file_location(module_name, str(module_path))
+                        if spec is None or spec.loader is None:
+                            raise ImportError("missing module spec loader")
 
-                try:
+                        module = importlib.util.module_from_spec(spec)
-                    if is_package:
+                        sys.modules[module_name] = module
-                        spec = importlib.util.spec_from_file_location(
+                        spec.loader.exec_module(module)
-                            module_name,
+                        self._external_modules.add(module_name)
-                            str(module_path),
+                        self._register_module(module)
-                            submodule_search_locations=[str(module_path.parent)],
+                    except Exception as exc:
-                        )
+                        log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr)
-                    else:
+            finally:
-                        spec = importlib.util.spec_from_file_location(module_name, str(module_path))
+                if path_added:
-                    if spec is None or spec.loader is None:
+                    try:
-                        raise ImportError("missing module spec loader")
+                        sys.path.remove(plugin_dir_str)
-
+                    except ValueError:
-                    module = importlib.util.module_from_spec(spec)
+                        pass
-                    sys.modules[module_name] = module
-                    spec.loader.exec_module(module)
-                    self._external_modules.add(module_name)
-                    self._register_module(module)
-                except Exception as exc:
-                    log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr)
 
     def discover(self) -> None:
         """Import and register plugins from the package."""

SYS/config.py

@@ -26,12 +26,16 @@ SCRIPT_DIR = Path(__file__).resolve().parent
 _SAVE_LOCK_DIRNAME = ".medios_save_lock"
 _SAVE_LOCK_TIMEOUT = 30.0  # seconds to wait for save lock
 _SAVE_LOCK_STALE_SECONDS = 3600  # consider lock stale after 1 hour
+_SAVE_LOCK_POLL_INTERVAL = 0.1  # seconds between lock acquisition attempts
+
+_WAL_CHECKPOINT_TIMEOUT = 5.0  # seconds for WAL checkpoint connection
 
 _CONFIG_CACHE: Dict[str, Any] = {}
 _LAST_SAVED_CONFIG: Dict[str, Any] = {}
 _CONFIG_SUMMARY_PENDING = False
 _CONFIG_SAVE_MAX_RETRIES = 5
 _CONFIG_SAVE_RETRY_DELAY = 0.15
+_CONFIG_SAVE_VERIFY_RETRIES = 3
 _CONFIG_MISSING = object()
 _PATH_ALIAS_TOKEN_RE = re.compile(r"^\$(?:\((?P<braced>[^)]+)\)|(?P<plain>[A-Za-z0-9_.-]+))$")
 
@@ -890,7 +894,7 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT):
                 logger.exception("Failed to inspect save lock directory %s: %s", lock_dir, exc)
             if time.time() - start > timeout:
                 raise ConfigSaveConflict("Save lock busy; could not acquire in time")
-            time.sleep(0.1)
+            time.sleep(_SAVE_LOCK_POLL_INTERVAL)
 
 
 def _release_save_lock(lock_dir: Path) -> None:
@@ -1030,10 +1034,10 @@ def save_config(config: Dict[str, Any]) -> int:
             # we don't contend with our main connection lock or active transactions.
             try:
                 try:
-                    with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
+                    with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con:
                         _con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
                 except Exception:
-                    with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
+                    with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con:
                         _con.execute("PRAGMA wal_checkpoint")
             except Exception as exc:
                 log(f"Warning: WAL checkpoint failed: {exc}")
@@ -1084,7 +1088,7 @@ def save(config: Dict[str, Any]) -> int:
     return save_config(config)
 
 
-def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: float = 0.15) -> int:
+def save_config_and_verify(config: Dict[str, Any], retries: int = _CONFIG_SAVE_VERIFY_RETRIES, delay: float = _CONFIG_SAVE_RETRY_DELAY) -> int:
     """Save configuration and verify crucial keys persisted to disk.
 
     This helper performs a best-effort verification loop that reloads the

SYS/database.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import atexit
 import sqlite3
 import json
 import ast
@@ -18,6 +19,8 @@ logger = logging.getLogger(__name__)
 # DB execute retry settings (for transient 'database is locked' errors)
 _DB_EXEC_RETRY_MAX = 5
 _DB_EXEC_RETRY_BASE_DELAY = 0.05
+_DB_CONNECT_TIMEOUT = 30.0
+_DB_BUSY_TIMEOUT_MS = 30000
 
 # The database is located in the project root (prefer explicit repo hints).
 def _resolve_root_dir() -> Path:
@@ -72,7 +75,7 @@ class Database:
         self.conn = sqlite3.connect(
             str(self.db_path),
             check_same_thread=False,
-            timeout=30.0  # Increase timeout to 30s to avoid locking issues
+            timeout=_DB_CONNECT_TIMEOUT
         )
         self.conn.row_factory = sqlite3.Row
         # Reentrant lock to allow nested DB calls within the same thread (e.g., transaction ->
@@ -82,7 +85,7 @@ class Database:
         # Use WAL mode for better concurrency (allows multiple readers + 1 writer)
         # Set a busy timeout so SQLite waits for short locks rather than immediately failing
         try:
-            self.conn.execute("PRAGMA busy_timeout = 30000")
+            self.conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
             self.conn.execute("PRAGMA journal_mode=WAL")
             self.conn.execute("PRAGMA synchronous=NORMAL")
         except sqlite3.Error:
@@ -282,23 +285,27 @@ class Database:
 # Singleton instance
 db = Database()
 
-_LOG_QUEUE: Queue[tuple[str, str, str]] = Queue()
+_LOG_QUEUE: Queue = Queue()
 _LOG_THREAD_STARTED = False
 _LOG_THREAD_LOCK = threading.Lock()
 _LOG_WRITE_COUNT = 0
 _LOG_PRUNE_INTERVAL = 500   # prune every N successful writes
 _LOG_MAX_AGE_DAYS = 7       # keep logs for this many days
+_LOG_WRITE_RETRY_MAX = 3
+_LOG_WRITE_RETRY_BASE_DELAY = 0.05
+_LOG_SENTINEL = object()
+_LOG_THREAD_REF: Optional[threading.Thread] = None
 
 
 def _ensure_log_db_schema() -> None:
     try:
         conn = sqlite3.connect(
             str(LOG_DB_PATH),
-            timeout=30.0,
+            timeout=_DB_CONNECT_TIMEOUT,
             check_same_thread=False,
         )
         try:
-            conn.execute("PRAGMA busy_timeout = 30000")
+            conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
             conn.execute("PRAGMA journal_mode=WAL")
             conn.execute("PRAGMA synchronous=NORMAL")
             conn.execute(
@@ -328,15 +335,20 @@ def _log_worker_loop() -> None:
     """
     global _LOG_WRITE_COUNT
     while True:
-        level, module, message = _LOG_QUEUE.get()
+        item = _LOG_QUEUE.get()
+        if item is _LOG_SENTINEL:
+            break
+        level, module, message = item
         try:
             attempts = 0
             written = False
-            while attempts < 3 and not written:
+            while attempts < _LOG_WRITE_RETRY_MAX and not written:
+                conn = None
+                cur = None
                 try:
-                    conn = sqlite3.connect(str(LOG_DB_PATH), timeout=30.0)
+                    conn = sqlite3.connect(str(LOG_DB_PATH), timeout=_DB_CONNECT_TIMEOUT)
                     try:
-                        conn.execute("PRAGMA busy_timeout = 30000")
+                        conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
                         conn.execute("PRAGMA journal_mode=WAL")
                         conn.execute("PRAGMA synchronous=NORMAL")
                     except sqlite3.Error:
@@ -352,13 +364,11 @@ def _log_worker_loop() -> None:
                             conn.commit()
                         except Exception:
                             pass
-                    cur.close()
-                    conn.close()
                     written = True
                 except sqlite3.OperationalError as exc:
                     attempts += 1
                     if 'locked' in str(exc).lower():
-                        time.sleep(0.05 * attempts)
+                        time.sleep(_LOG_WRITE_RETRY_BASE_DELAY * attempts)
                         continue
                     # Non-lock operational errors: abort attempts
                     log(f"Warning: Failed to write log entry (operational): {exc}")
@@ -366,6 +376,17 @@ def _log_worker_loop() -> None:
                 except Exception as exc:
                     log(f"Warning: Failed to write log entry: {exc}")
                     break
+                finally:
+                    if cur is not None:
+                        try:
+                            cur.close()
+                        except Exception:
+                            pass
+                    if conn is not None:
+                        try:
+                            conn.close()
+                        except Exception:
+                            pass
             if not written:
                 # Fallback to a file-based log so we never lose the message silently
                 try:
@@ -399,7 +420,7 @@ def _log_worker_loop() -> None:
 
 
 def _ensure_log_thread() -> None:
-    global _LOG_THREAD_STARTED
+    global _LOG_THREAD_STARTED, _LOG_THREAD_REF
     if _LOG_THREAD_STARTED:
         return
     with _LOG_THREAD_LOCK:
@@ -411,8 +432,24 @@ def _ensure_log_thread() -> None:
             daemon=True
         )
         thread.start()
+        _LOG_THREAD_REF = thread
         _LOG_THREAD_STARTED = True
 
+
+def _shutdown_log_thread() -> None:
+    global _LOG_THREAD_REF
+    _LOG_QUEUE.put(_LOG_SENTINEL)
+    thread = _LOG_THREAD_REF
+    if thread is not None:
+        try:
+            thread.join(timeout=3.0)
+        except Exception:
+            pass
+        _LOG_THREAD_REF = None
+
+
+atexit.register(_shutdown_log_thread)
+
 def get_db() -> Database:
     return db
 
@@ -533,7 +570,11 @@ def get_config_all() -> Dict[str, Any]:
 
 # Worker Management Methods for medios.db
 
-def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection:
+_WORKER_DB_DEFAULT_TIMEOUT = 0.75
+_WORKER_DB_DEFAULT_RETRIES = 1
+_WORKER_DB_RETRY_BASE_DELAY = 0.05
+
+def _worker_db_connect(timeout: float = _WORKER_DB_DEFAULT_TIMEOUT) -> sqlite3.Connection:
     conn = sqlite3.connect(
         str(DB_PATH),
         timeout=timeout,
@@ -555,8 +596,8 @@ def _worker_db_execute(
     params: tuple = (),
     *,
     fetch: Optional[str] = None,
-    timeout: float = 0.75,
+    timeout: float = _WORKER_DB_DEFAULT_TIMEOUT,
-    retries: int = 1,
+    retries: int = _WORKER_DB_DEFAULT_RETRIES,
 ) -> Any:
     attempts = 0
     while True:
@@ -580,7 +621,7 @@ def _worker_db_execute(
             msg = str(exc).lower()
             if "locked" in msg and attempts < retries:
                 attempts += 1
-                time.sleep(0.05 * attempts)
+                time.sleep(_WORKER_DB_RETRY_BASE_DELAY * attempts)
                 continue
             raise
         finally:

SYS/file_server.py

@@ -1,9 +1,11 @@
 """Simple HTTP file server for serving files in web mode."""
 
+import atexit
+import os
 import threading
 import socket
 import logging
-from http.server import HTTPServer, SimpleHTTPRequestHandler
+from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
 from pathlib import Path
 from typing import Optional
 import mimetypes
@@ -11,9 +13,12 @@ import urllib.parse
 
 logger = logging.getLogger(__name__)
 
+_DNS_FALLBACK_HOST = os.environ.get("MM_DNS_SERVER", "8.8.8.8")
+_DNS_FALLBACK_PORT = 80
+
 # Global server instance
-_file_server: Optional[HTTPServer] = None
+_file_server: Optional[ThreadingHTTPServer] = None
-_server_thread: Optional[threading.Thread] = None
+_file_server_thread: Optional[threading.Thread] = None
 _server_port: int = 8001
 
 
@@ -84,7 +89,7 @@ def get_local_ip() -> Optional[str]:
     try:
         # Connect to a remote server to determine local IP
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect(("8.8.8.8", 80))
+        s.connect((_DNS_FALLBACK_HOST, _DNS_FALLBACK_PORT))
         ip = s.getsockname()[0]
         s.close()
         return ip
@@ -116,7 +121,7 @@ def start_file_server(port: int = 8001) -> Optional[str]:
 
         # Create server
         server_address = ("", port)
-        _file_server = HTTPServer(server_address, FileServerHandler)
+        _file_server = ThreadingHTTPServer(server_address, FileServerHandler)
 
         # Start in daemon thread
         _server_thread = threading.Thread(
@@ -125,6 +130,8 @@ def start_file_server(port: int = 8001) -> Optional[str]:
         )
         _server_thread.start()
 
+        atexit.register(stop_file_server)
+
         logger.info(f"File server started on port {port}")
 
         # Get local IP

SYS/metadata.py

@@ -523,8 +523,8 @@ def _read_sidecar_metadata(
         lower = line.lower()
         if lower.startswith("hash:"):
             hash_value = line.split(":", 1)[1].strip() if ":" in line else ""
-        elif lower.startswith("url:") or lower.startswith("url:"):
+        elif lower.startswith("url:"):
-            # Parse url (handle legacy 'url:' format)
+            # Parse url
             url_part = line.split(":", 1)[1].strip() if ":" in line else ""
             if url_part:
                 for url_segment in url_part.split(","):
@@ -638,44 +638,7 @@ def write_tags(
             return
         except Exception as e:
             debug(f"Failed to add tags to database: {e}", file=sys.stderr)
-            # Fall through to sidecar creation as fallback
+            return
-
-    # Create sidecar path
-    try:
-        sidecar = media_path.parent / (media_path.name + ".tag")
-    except Exception:
-        sidecar = media_path.with_name(media_path.name + ".tag")
-
-    # Handle edge case: empty/invalid base name
-    try:
-        if not sidecar.stem or sidecar.name in {".tag",
-                                                "-.tag",
-                                                "_.tag"}:
-            fallback_base = (
-                media_path.stem
-                or _sanitize_title_for_filename(extract_title(tag_list) or "")
-                or "untitled"
-            )
-            sidecar = media_path.parent / f"{fallback_base}.tag"
-    except Exception:
-        logger.exception("Failed to determine fallback .tag sidecar base for %s", media_path)
-
-    # Write via consolidated function
-    try:
-        lines: List[str] = []
-        lines.extend(str(tag).strip().lower() for tag in tag_list if str(tag).strip())
-
-        if lines:
-            sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
-            if emit_debug:
-                debug(f"Tags: {sidecar}")
-        else:
-            try:
-                sidecar.unlink()
-            except FileNotFoundError:
-                pass
-    except OSError as exc:
-        debug(f"Failed to write tag sidecar {sidecar}: {exc}", file=sys.stderr)
 
 
 def write_metadata(
@@ -729,45 +692,7 @@ def write_metadata(
             return
         except Exception as e:
             debug(f"Failed to add metadata to database: {e}", file=sys.stderr)
-            # Fall through to sidecar creation as fallback
+            return
-
-    # Create sidecar path
-    try:
-        sidecar = media_path.parent / (media_path.name + ".metadata")
-    except Exception:
-        sidecar = media_path.with_name(media_path.name + ".metadata")
-
-    try:
-        lines = []
-
-        # Add hash if available
-        if hash_value:
-            lines.append(f"hash:{hash_value}")
-
-        # Add known url
-        for url in url_list:
-            if str(url).strip():
-                clean = str(url).strip()
-                lines.append(f"url:{clean}")
-
-        # Add relationships
-        for rel in rel_list:
-            if str(rel).strip():
-                lines.append(f"relationship:{str(rel).strip()}")
-
-        # Write metadata file
-        if lines:
-            sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
-            if emit_debug:
-                debug(f"Wrote metadata to {sidecar}")
-        else:
-            # Remove if no content
-            try:
-                sidecar.unlink()
-            except FileNotFoundError:
-                pass
-    except OSError as exc:
-        debug(f"Failed to write metadata sidecar {sidecar}: {exc}", file=sys.stderr)
 
 
 def extract_title(tags: Iterable[str]) -> Optional[str]:
@@ -2864,17 +2789,7 @@ def build_ffmpeg_command(
             "192k",
         ])
         cmd.extend(["-f", "opus"])
-    elif fmt == "audio":
+    if fmt not in ("mp4", "webm", "mp3", "flac", "wav", "aac", "m4a", "ogg", "opus", "copy"):
-        # Legacy format name for mp3
-        cmd.extend([
-            "-vn",
-            "-c:a",
-            "libmp3lame",
-            "-b:a",
-            "192k",
-        ])
-        cmd.extend(["-f", "mp3"])
-    elif fmt != "copy":
         raise ValueError(f"Unsupported format: {fmt}")
 
     cmd.append(str(output_path))

SYS/result_table.py

@@ -45,22 +45,6 @@ def _rich():
     return _rich_mod
 
 
-# Optional Textual imports - lazily loaded to avoid pulling in ~300ms of textual
-# at import time when the TUI is not being used.
-import importlib.util as _importlib_util
-TEXTUAL_AVAILABLE: bool = _importlib_util.find_spec("textual") is not None
-
-# Tree is populated lazily on first call to build_metadata_tree().
-_textual_Tree: Any = None
-
-
-def _get_textual_Tree() -> Any:
-    global _textual_Tree
-    if _textual_Tree is None:
-        from textual.widgets import Tree as _Tree
-        _textual_Tree = _Tree
-    return _textual_Tree
-
 
 # Import ResultModel from the API for typing; avoid runtime redefinition issues
 from typing import TYPE_CHECKING
@@ -69,6 +53,8 @@ if TYPE_CHECKING:
 else:
     ResultModel = None  # type: ignore[assignment]
 
+
+
 # Reuse the existing format_bytes helper under a clearer alias
 from SYS.utils import format_bytes as format_mb
 
@@ -587,30 +573,6 @@ class InputOption:
         }
 
 
-@dataclass
-class TUIResultCard:
-    """Represents a result as a UI card with title, metadata, and actions.
-
-    Used in hub-ui and TUI contexts to render individual search results
-    as grouped components with visual structure.
-    """
-
-    title: str
-    subtitle: Optional[str] = None
-    metadata: Optional[Dict[str, str]] = None
-    media_kind: Optional[str] = None
-    tag: Optional[List[str]] = None
-    file_hash: Optional[str] = None
-    file_size: Optional[str] = None
-    duration: Optional[str] = None
-
-    def __post_init__(self):
-        """Initialize default values."""
-        if self.metadata is None:
-            self.metadata = {}
-        if self.tag is None:
-            self.tag = []
-
 
 @dataclass
 class Column:
@@ -2113,122 +2075,6 @@ class Table:
             return self.rows[idx]
         return None
 
-    # TUI-specific formatting methods
-
-    def to_datatable_rows(self, source: str = "unknown") -> List[List[str]]:
-        """Convert results to rows suitable for Textual DataTable widget.
-
-        Args:
-            source: Source type for formatting context (openlibrary, soulseek, etc.)
-
-        Returns:
-            List of row value lists
-        """
-        rows = []
-        for result in self.rows:
-            row_values = self._format_datatable_row(result, source)
-            rows.append(row_values)
-        return rows
-
-    def _format_datatable_row(self,
-                              row: Row,
-                              source: str = "unknown") -> List[str]:
-        """Format a ResultRow for DataTable display.
-
-        Args:
-            row: ResultRow to format
-            source: Source type
-
-        Returns:
-            List of column values as strings
-        """
-        # Extract values from row columns
-        values = [col.value for col in row.columns]
-
-        # Truncate to reasonable lengths for table display
-        return [v[:60] if len(v) > 60 else v for v in values]
-
-    def to_result_cards(self) -> List[TUIResultCard]:
-        """Convert all rows to TUIResultCard objects for card-based UI display.
-
-        Returns:
-            List of TUIResultCard objects
-        """
-        cards = []
-        for row in self.rows:
-            card = self._row_to_card(row)
-            cards.append(card)
-        return cards
-
-    def _row_to_card(self, row: Row) -> TUIResultCard:
-        """Convert a ResultRow to a TUIResultCard.
-
-        Args:
-            row: ResultRow to convert
-
-        Returns:
-            TUIResultCard with extracted metadata
-        """
-        # Build metadata dict from row columns
-        metadata = {}
-        title = ""
-
-        for col in row.columns:
-            if col.name.lower() == "title":
-                title = col.value
-            metadata[col.name] = col.value
-
-        # Extract tag values if present
-        tag = []
-        if "Tag" in metadata:
-            tag_val = metadata["Tag"]
-            if tag_val:
-                tag = [t.strip() for t in tag_val.split(",")][:5]
-
-        # Try to find useful metadata fields
-        subtitle = metadata.get("Artist", metadata.get("Author", ""))
-        media_kind = metadata.get("Type", metadata.get("Media Kind", ""))
-        file_size = metadata.get("Size", "")
-        duration = metadata.get("Duration", "")
-        file_hash = metadata.get("Hash", "")
-
-        return TUIResultCard(
-            title=title or "Unknown",
-            subtitle=subtitle,
-            metadata=metadata,
-            media_kind=media_kind,
-            tag=tag,
-            file_hash=file_hash or None,
-            file_size=file_size or None,
-            duration=duration or None,
-        )
-
-    def build_metadata_tree(self, tree_widget: "Tree") -> None:
-        """Populate a Textual Tree widget with result metadata hierarchy.
-
-        Args:
-            tree_widget: Textual Tree widget to populate
-
-        Raises:
-            ImportError: If Textual not available
-        """
-        if not TEXTUAL_AVAILABLE:
-            raise ImportError("Textual not available for tree building")
-
-        tree_widget.reset(self.title or "Results")
-        root = tree_widget.root
-
-        # Add each row as a top-level node
-        for i, row in enumerate(self.rows, 1):
-            row_node = root.add(f"[bold]Result {i}[/bold]")
-
-            # Add columns as children
-            for col in row.columns:
-                value_str = col.value
-                if len(value_str) > 100:
-                    value_str = value_str[:97] + "..."
-                row_node.add_leaf(f"[cyan]{col.name}[/cyan]: {value_str}")
-
 
 def _format_size(size: Any, integer_only: bool = False) -> str:
     """Format file size as human-readable string.
@@ -2577,9 +2423,46 @@ class ItemDetailView(Table):
                 return f"{label[:max_len - 3]}..."
             return label
 
+        def _looks_like_local_path(value: Any) -> bool:
+            text = str(value or "").strip()
+            if not text:
+                return False
+            try:
+                p = Path(text)
+                if p.exists():
+                    return True
+                return text.startswith("/") or text.startswith("\\\\") or (len(text) >= 2 and text[1] == ":" and text[2:3] in ("\\", "/"))
+            except Exception:
+                return False
+
+        def _short_path_label(path_value: str, *, max_len: int = 68) -> str:
+            text = str(path_value or "").strip()
+            if not text:
+                return ""
+            if len(text) <= max_len:
+                return text
+            parts = text.replace("\\", "/").rstrip("/").split("/")
+            if len(parts) >= 3:
+                head = parts[0]
+                tail = parts[-1]
+                mid = "..."
+                label = f"{head}/{mid}/{tail}"
+                if len(label) > max_len:
+                    return f"{text[:max_len - 3]}..."
+                return label
+            return f"{text[:max_len - 3]}..."
+
         def _render_detail_value(key: str, value: Any) -> Any:
-            # Keep terminal links compact while preserving full click target.
+            key_lower = str(key or "").strip().lower()
-            if str(key or "").strip().lower() in {"path", "url"} and _looks_like_http_url(value):
+            if key_lower == "path" and _looks_like_local_path(value):
+                full_path = str(value).strip()
+                try:
+                    file_uri = Path(full_path).as_uri()
+                except Exception:
+                    return str(value)
+                label = _short_path_label(full_path)
+                return _rich().Text(label, style=f"underline cyan link {file_uri}")
+            if key_lower in {"path", "url"} and _looks_like_http_url(value):
                 full_url = str(value).strip()
                 label = _short_link_label(full_url)
                 return _rich().Text(label, style=f"underline cyan link {full_url}")

SYS/rich_display.py

@@ -109,7 +109,7 @@ def show_plugin_config_panel(
     group = Group(
         Text("The following plugins are not configured and cannot be used:\n"),
         table,
-        Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.")
+        Text.from_markup("\nTo configure them, use the [bold cyan].config[/bold cyan] command in the REPL.")
     )
 
     panel = Panel(

SYS/worker_manager.py

@@ -260,6 +260,11 @@ class WorkerLoggingHandler(logging.StreamHandler):
         super().close()
 
 
+_DEFAULT_STDOUT_FLUSH_BYTES = 4096
+_DEFAULT_STDOUT_FLUSH_INTERVAL = 0.75
+_REFRESH_THREAD_JOIN_TIMEOUT = 5
+
+
 class WorkerManager:
     """Manages persistent worker tasks using the central medios.db."""
 
@@ -284,8 +289,8 @@ class WorkerManager:
         self._stdout_buffer_sizes: Dict[Tuple[str, str], int] = {}
         self._stdout_buffer_steps: Dict[Tuple[str, str], Optional[str]] = {}
         self._stdout_last_flush: Dict[Tuple[str, str], float] = {}
-        self._stdout_flush_bytes = 4096
+        self._stdout_flush_bytes = _DEFAULT_STDOUT_FLUSH_BYTES
-        self._stdout_flush_interval = 0.75
+        self._stdout_flush_interval = _DEFAULT_STDOUT_FLUSH_INTERVAL
 
 
     def __enter__(self):
@@ -707,7 +712,7 @@ class WorkerManager:
         self._stop_refresh = True
         self._refresh_stop_event.set()
         if self.refresh_thread:
-            self.refresh_thread.join(timeout=5)
+            self.refresh_thread.join(timeout=_REFRESH_THREAD_JOIN_TIMEOUT)
             self.refresh_thread = None
 
     def _start_refresh_if_needed(self) -> None:

cmdlet/_display.py

@@ -0,0 +1,470 @@
+"""UI/display helpers for Rich output, saved-output panels, and result persistence.
+
+Contains utilities for:
+- Printing to stderr safely without disrupting Rich Live display
+- Rendering saved output panels when ``-path`` is used
+- Persisting items for ``@N`` selection across command boundaries
+- Formatting byte sizes for display
+- Moving emitted temp/PATH artifacts to user-specified destinations
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from contextlib import AbstractContextManager, nullcontext
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence
+
+from SYS.logger import log
+from SYS.result_publication import publish_result_table
+from SYS.result_table import Table
+from SYS.rich_display import stderr_console as get_stderr_console
+from ._pipeobject_utils import get_field, get_pipe_object_path
+
+__all__ = [
+    "_print_live_safe_stderr",
+    "_print_saved_output_panel",
+    "_apply_saved_path_update",
+    "_unique_destination_path",
+    "_extract_flag_value",
+    "apply_output_path_from_pipeobjects",
+    "display_and_persist_items",
+    "fmt_bytes",
+]
+
+
+def fmt_bytes(n: Optional[int]) -> str:
+    """Format bytes as human-readable with 1 decimal place (MB/GB).
+
+    Args:
+            n: Number of bytes, or None
+
+    Returns:
+            Formatted string like "1.5 MB" or "2.0 GB", or "unknown"
+    """
+    if n is None or n < 0:
+        return "unknown"
+    gb = n / (1024.0 * 1024.0 * 1024.0)
+    if gb >= 1.0:
+        return f"{gb:.1f} GB"
+    mb = n / (1024.0 * 1024.0)
+    return f"{mb:.1f} MB"
+
+
+def _extract_flag_value(args: Sequence[str], *flags: str) -> Optional[str]:
+    """Return the value for the first matching flag in args.
+
+    This is intentionally lightweight (no cmdlet spec required) so callers in CLI/pipeline
+    can share the same behavior.
+    """
+    if not args:
+        return None
+    want = {str(f).strip().lower()
+            for f in flags if str(f).strip()}
+    if not want:
+        return None
+    try:
+        tokens = [str(a) for a in args]
+    except Exception:
+        tokens = list(args)  # type: ignore[list-item]
+    for i, tok in enumerate(tokens):
+        low = str(tok).strip().lower()
+        if low in want:
+            if i + 1 >= len(tokens):
+                return None
+            nxt = str(tokens[i + 1])
+            if not nxt.strip():
+                return None
+            if nxt.startswith("-"):
+                return None
+            return nxt
+    return None
+
+
+def _unique_destination_path(dest: Path) -> Path:
+    """Generate a non-colliding destination path by appending " (N)"."""
+    try:
+        if not dest.exists():
+            return dest
+    except Exception:
+        return dest
+
+    parent = dest.parent
+    stem = dest.stem
+    suffix = dest.suffix
+    for i in range(1, 10_000):
+        candidate = parent / f"{stem} ({i}){suffix}"
+        try:
+            if not candidate.exists():
+                return candidate
+        except Exception:
+            return candidate
+    return dest
+
+
+def _print_live_safe_stderr(message: str) -> None:
+    """Print to stderr without breaking Rich Live progress output."""
+    try:
+        from SYS.rich_display import stderr_console  # type: ignore
+    except Exception:
+        return
+
+    cm: AbstractContextManager[Any] | None = None
+    try:
+        from SYS import pipeline as _pipeline_ctx  # type: ignore
+
+        suspend = getattr(_pipeline_ctx, "suspend_live_progress", None)
+        candidate = suspend() if callable(suspend) else None
+        if isinstance(candidate, AbstractContextManager):
+            cm = candidate
+        elif candidate is not None and hasattr(candidate, "__enter__") and hasattr(candidate, "__exit__"):
+            cm = candidate  # type: ignore[arg-type]
+    except Exception:
+        cm = None
+
+    if cm is None:
+        cm = nullcontext()
+
+    try:
+        console = stderr_console()
+        print_func = getattr(console, "print", None)
+    except Exception:
+        return
+
+    if not callable(print_func):
+        return
+
+    try:
+        with cm:
+            print_func(str(message))
+    except Exception:
+        return
+
+
+def _print_saved_output_panel(item: Any, final_path: Path) -> None:
+    """When -path is used, print a Rich panel summarizing the saved output.
+
+    Shows: Title, Location, Hash.
+    Best-effort: reads existing fields first to avoid recomputing hashes.
+    """
+    try:
+        from rich.panel import Panel  # type: ignore
+        from rich.table import Table as RichTable  # type: ignore
+        from SYS.rich_display import stderr_console  # type: ignore
+    except Exception:
+        return
+
+    cm: AbstractContextManager[Any] | None = None
+    try:
+        from SYS import pipeline as _pipeline_ctx  # type: ignore
+
+        suspend = getattr(_pipeline_ctx, "suspend_live_progress", None)
+        cm_candidate = suspend() if callable(suspend) else None
+        if isinstance(cm_candidate, AbstractContextManager):
+            cm = cm_candidate
+        elif cm_candidate is not None and hasattr(cm_candidate, "__enter__") and hasattr(cm_candidate, "__exit__"):
+            cm = cm_candidate  # type: ignore[arg-type]
+    except Exception:
+        cm = None
+
+    if cm is None:
+        cm = nullcontext()
+
+    try:
+        location = str(final_path)
+    except Exception:
+        location = ""
+
+    title = ""
+    try:
+        title = str(get_field(item, "title") or get_field(item, "name") or "").strip()
+    except Exception:
+        title = ""
+    if not title:
+        try:
+            title = str(final_path.stem or final_path.name)
+        except Exception:
+            title = ""
+
+    file_hash = ""
+    try:
+        file_hash = str(get_field(item, "hash") or get_field(item, "sha256") or "").strip()
+    except Exception:
+        file_hash = ""
+    if not file_hash:
+        try:
+            from SYS.utils import sha256_file  # type: ignore
+
+            file_hash = str(sha256_file(final_path) or "").strip()
+        except Exception:
+            file_hash = ""
+
+    grid = RichTable.grid(padding=(0, 1))
+    grid.add_column(justify="right", style="bold")
+    grid.add_column()
+    grid.add_row("Title", title or "(unknown)")
+    grid.add_row("Location", location or "(unknown)")
+    grid.add_row("Hash", file_hash or "(unknown)")
+
+    try:
+        console = stderr_console()
+        print_func = getattr(console, "print", None)
+    except Exception:
+        return
+
+    if not callable(print_func):
+        return
+
+    try:
+        with cm:
+            print_func(Panel(grid, title="Saved", expand=False))
+    except Exception:
+        return
+
+
+def _apply_saved_path_update(item: Any, *, old_path: str, new_path: str) -> None:
+    """Update a PipeObject-like item after its backing file has moved."""
+    old_str = str(old_path)
+    new_str = str(new_path)
+    if isinstance(item, dict):
+        try:
+            if str(item.get("path") or "") == old_str:
+                item["path"] = new_str
+        except Exception:
+            pass
+        try:
+            if str(item.get("target") or "") == old_str:
+                item["target"] = new_str
+        except Exception:
+            pass
+        try:
+            extra = item.get("extra")
+            if isinstance(extra, dict):
+                if str(extra.get("target") or "") == old_str:
+                    extra["target"] = new_str
+                if str(extra.get("path") or "") == old_str:
+                    extra["path"] = new_str
+        except Exception:
+            pass
+        return
+
+    try:
+        if getattr(item, "path", None) == old_str:
+            setattr(item, "path", new_str)
+    except Exception:
+        pass
+    try:
+        extra = getattr(item, "extra", None)
+        if isinstance(extra, dict):
+            if str(extra.get("target") or "") == old_str:
+                extra["target"] = new_str
+            if str(extra.get("path") or "") == old_str:
+                extra["path"] = new_str
+    except Exception:
+        pass
+
+
+def apply_output_path_from_pipeobjects(
+    *,
+    cmd_name: str,
+    args: Sequence[str],
+    emits: Sequence[Any],
+) -> List[Any]:
+    """If the user supplied `-path`, move emitted temp/PATH files there.
+
+    This enables a dynamic pattern:
+    - Any cmdlet can include `SharedArgs.PATH`.
+    - If it emits a file-backed PipeObject (`path` exists on disk) and the item is
+      a temp/PATH artifact, then `-path <dest>` will save it to that location.
+
+    Rules:
+    - Only affects items whose `action` matches the current cmdlet.
+    - Only affects items that look like local artifacts (`is_temp` True or `store` == PATH).
+    - Updates the emitted object's `path` (and `target` when it points at the same file).
+    """
+    dest_raw = _extract_flag_value(args, "-path", "--path")
+    if not dest_raw:
+        return list(emits or [])
+
+    try:
+        dest_str = str(dest_raw).strip()
+        if "://" in dest_str:
+            _print_live_safe_stderr(
+                f"Ignoring -path value that looks like a URL: {dest_str}"
+            )
+            return list(emits or [])
+    except Exception:
+        pass
+
+    cmd_norm = str(cmd_name or "").replace("_", "-").strip().lower()
+    if not cmd_norm:
+        return list(emits or [])
+
+    try:
+        dest_hint_dir = str(dest_raw).endswith(("/", "\\"))
+    except Exception:
+        dest_hint_dir = False
+
+    try:
+        dest_path = Path(str(dest_raw)).expanduser()
+    except Exception:
+        return list(emits or [])
+
+    items = list(emits or [])
+    artifact_indices: List[int] = []
+    artifact_paths: List[Path] = []
+    for idx, item in enumerate(items):
+        action = str(get_field(item, "action", "") or "").strip().lower()
+        if not action.startswith("cmdlet:"):
+            continue
+        action_name = action.split(":", 1)[-1].strip().lower()
+        if action_name != cmd_norm:
+            continue
+
+        store = str(get_field(item, "store", "") or "").strip().lower()
+        is_temp = bool(get_field(item, "is_temp", False))
+        if not (is_temp or store == "path"):
+            continue
+
+        src_str = get_pipe_object_path(item)
+        if not src_str:
+            continue
+        try:
+            src = Path(str(src_str)).expanduser()
+        except Exception:
+            continue
+        try:
+            if not src.exists() or not src.is_file():
+                continue
+        except Exception:
+            continue
+
+        artifact_indices.append(idx)
+        artifact_paths.append(src)
+
+    if not artifact_indices:
+        return items
+
+    if len(artifact_indices) > 1:
+        if dest_path.suffix:
+            dest_dir = dest_path.parent
+        else:
+            dest_dir = dest_path
+        try:
+            dest_dir.mkdir(parents=True, exist_ok=True)
+        except Exception as exc:
+            _print_live_safe_stderr(
+                f"Failed to create destination directory: {dest_dir} ({exc})"
+            )
+            return items
+
+        for idx, src in zip(artifact_indices, artifact_paths):
+            final = dest_dir / src.name
+            final = _unique_destination_path(final)
+            try:
+                if src.resolve() == final.resolve():
+                    _apply_saved_path_update(
+                        items[idx],
+                        old_path=str(src),
+                        new_path=str(final),
+                    )
+                    _print_saved_output_panel(items[idx], final)
+                    continue
+            except Exception:
+                pass
+            try:
+                shutil.move(str(src), str(final))
+            except Exception as exc:
+                _print_live_safe_stderr(f"Failed to save output to {final}: {exc}")
+                continue
+            _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final))
+            _print_saved_output_panel(items[idx], final)
+
+        return items
+
+    src = artifact_paths[0]
+    idx = artifact_indices[0]
+    final: Path
+    try:
+        if dest_hint_dir or (dest_path.exists() and dest_path.is_dir()):
+            final = dest_path / src.name
+        else:
+            final = dest_path
+    except Exception:
+        final = dest_path
+
+    try:
+        final.parent.mkdir(parents=True, exist_ok=True)
+    except Exception as exc:
+        _print_live_safe_stderr(
+            f"Failed to create destination directory: {final.parent} ({exc})"
+        )
+        return items
+
+    final = _unique_destination_path(final)
+    try:
+        if src.resolve() != final.resolve():
+            shutil.move(str(src), str(final))
+    except Exception as exc:
+        _print_live_safe_stderr(f"Failed to save output to {final}: {exc}")
+        return items
+
+    _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final))
+    _print_saved_output_panel(items[idx], final)
+    return items
+
+
+def display_and_persist_items(
+    items: List[Any],
+    title: str = "Result",
+    subject: Optional[Any] = None,
+    display_type: str = "item",
+    table: Optional[Table] = None,
+) -> None:
+    """Display detail panels and persist items for @N selection.
+
+    This helper function:
+    1. Renders individual detail panels for each item
+    2. Creates a result table with all items (or uses provided table)
+    3. Persists the table so @N selection works across command boundaries
+
+    Args:
+        items: List of items/dicts to display and make selectable
+        title: Title for the result table (default: "Result")
+        subject: Optional subject to associate with the results (usually first item or list)
+        display_type: Type of display - "item" (default), "tag", or "custom" (no panels)
+        table: Optional pre-built Table object (if None, creates new Table with title)
+    """
+    if not items:
+        return
+
+    try:
+        from SYS import pipeline as pipeline_context
+
+        if table is None:
+            table = Table(title=title)
+
+        if display_type != "custom":
+            try:
+                from SYS.rich_display import render_item_details_panel
+
+                panel_prefix = "Tag" if display_type == "tag" else "Item"
+
+                for idx, item in enumerate(items, 1):
+                    try:
+                        render_item_details_panel(item, title=f"#{idx} {panel_prefix} Details")
+                    except Exception:
+                        pass
+            except Exception:
+                pass
+
+        if not getattr(table, "_items_added", False):
+            for item in items:
+                table.add_result(item)
+
+        setattr(table, "_rendered_by_cmdlet", True)
+
+        publish_result_table(pipeline_context, table, items, subject=subject)
+    except Exception:
+        pass

cmdlet/_pipeobject_utils.py

@@ -0,0 +1,686 @@
+"""PipeObject utilities for chainable cmdlet and multi-action pipelines.
+
+Contains helpers for creating, normalizing, mutating, and extracting data from
+PipeObject instances and result dicts. Also includes the canonical
+``merge_sequences`` dedup utility used across cmdlets.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable as IterableABC
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from SYS import models
+from SYS.item_accessors import get_field as _item_accessor_get_field
+from SYS.payload_builders import build_file_result_payload
+from ._store_utils import resolve_hash_for_cmdlet
+
+__all__ = [
+    "get_field",
+    "merge_sequences",
+    "normalize_result_input",
+    "normalize_result_items",
+    "value_has_content",
+    "filter_results_by_temp",
+    "create_pipe_object_result",
+    "mark_as_temp",
+    "set_parent_hash",
+    "get_pipe_object_path",
+    "get_pipe_object_hash",
+    "coerce_to_pipe_object",
+    "resolve_item_store_hash",
+    "propagate_metadata",
+    "extract_relationships",
+    "extract_duration",
+    "pipeline_item_local_path",
+]
+
+
+def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any:
+    """Extract a field from either a dict or object with fallback default.
+
+    Handles both dict.get(field) and getattr(obj, field) access patterns.
+    Also handles lists by accessing the first element.
+    For PipeObjects, checks the extra field as well.
+    Used throughout cmdlet to uniformly access fields from mixed types.
+
+    Args:
+            obj: Dict, object, or list to extract from
+            field: Field name to retrieve
+            default: Value to return if field not found (default: None)
+
+    Returns:
+            Field value if found, otherwise the default value
+
+    Examples:
+            get_field(result, "hash")          # From dict or object
+            get_field(result, "table", "unknown")  # With default
+    """
+    return _item_accessor_get_field(obj, field, default)
+
+
+def merge_sequences(*sources: Optional[Iterable[Any]],
+                    case_sensitive: bool = True) -> list[str]:
+    """Merge iterable sources while preserving order and removing duplicates."""
+    seen: set[str] = set()
+    merged: list[str] = []
+    for source in sources:
+        if not source:
+            continue
+        if isinstance(source, str) or not isinstance(source, IterableABC):
+            iterable = [source]
+        else:
+            iterable = source
+        for value in iterable:
+            if value is None:
+                continue
+            text = str(value).strip()
+            if not text:
+                continue
+            key = text if case_sensitive else text.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            merged.append(text)
+    return merged
+
+
+def normalize_result_input(result: Any) -> List[Dict[str, Any]]:
+    """Normalize input result to a list of dicts.
+
+    Handles:
+    - None -> []
+    - Dict -> [dict]
+    - List of dicts -> list as-is
+    - PipeObject -> [dict]
+    - List of PipeObjects -> list of dicts
+
+    Args:
+            result: Result from piped input
+
+    Returns:
+            List of result dicts (may be empty)
+    """
+    if result is None:
+        return []
+
+    if isinstance(result, dict):
+        return [result]
+
+    if isinstance(result, list):
+        output = []
+        for item in result:
+            if isinstance(item, dict):
+                output.append(item)
+            elif hasattr(item, "to_dict"):
+                output.append(item.to_dict())
+            else:
+                output.append(item)
+        return output
+
+    if hasattr(result, "to_dict"):
+        return [result.to_dict()]
+
+    if isinstance(result, dict):
+        return [result]
+
+    return []
+
+
+def normalize_result_items(
+    result: Any,
+    *,
+    include_falsey_single: bool = False,
+) -> List[Any]:
+    """Normalize piped input to a raw item list without converting item types."""
+    if isinstance(result, list):
+        return list(result)
+    if result is None:
+        return []
+    if include_falsey_single or result:
+        return [result]
+    return []
+
+
+def value_has_content(value: Any) -> bool:
+    """Return True when a value should be treated as present for payload building."""
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return bool(value.strip())
+    if isinstance(value, (list, tuple, set)):
+        return len(value) > 0
+    return True
+
+
+def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]:
+    """Filter results by temporary status.
+
+    Args:
+            results: List of result dicts or PipeObjects
+            include_temp: If True, keep temp files; if False, exclude them
+
+    Returns:
+            Filtered list
+    """
+    if include_temp:
+        return results
+
+    filtered = []
+    for result in results:
+        is_temp = False
+
+        if hasattr(result, "is_temp"):
+            is_temp = result.is_temp
+        elif isinstance(result, dict):
+            is_temp = result.get("is_temp", False)
+
+        if not is_temp:
+            filtered.append(result)
+
+    return filtered
+
+
+def create_pipe_object_result(
+    source: str,
+    identifier: str,
+    file_path: str,
+    cmdlet_name: str,
+    title: Optional[str] = None,
+    hash_value: Optional[str] = None,
+    is_temp: bool = False,
+    parent_hash: Optional[str] = None,
+    tag: Optional[List[str]] = None,
+    **extra: Any,
+) -> Dict[str, Any]:
+    """Create a PipeObject-compatible result dict for pipeline chaining.
+
+    This is a helper to emit results in the standard format that downstream
+    cmdlet can process (filter, tag, cleanup, etc.).
+
+    Args:
+            source: Source system (e.g., 'local', 'hydrus', 'download')
+            identifier: Unique ID from source
+            file_path: Path to the file
+            cmdlet_name: Name of the cmdlet that created this (e.g., 'download-data', 'screen-shot')
+            title: Human-readable title
+            hash_value: SHA-256 hash of file (for integrity)
+            is_temp: If True, this is a temporary/intermediate artifact
+            parent_hash: Hash of the parent file in the chain (for provenance)
+            tag: List of tag values to apply
+            **extra: Additional fields
+
+    Returns:
+            Dict with all PipeObject fields for emission
+    """
+    store_override = extra.pop("store", None)
+
+    result = build_file_result_payload(
+        title=title,
+        path=file_path,
+        hash_value=hash_value,
+        store=store_override if store_override is not None else source,
+        tag=tag,
+        source=source,
+        id=identifier,
+        action=f"cmdlet:{cmdlet_name}",
+        **extra,
+    )
+
+    if is_temp:
+        result["is_temp"] = True
+    if parent_hash:
+        result["parent_hash"] = parent_hash
+
+    return result
+
+
+def mark_as_temp(pipe_object: Dict[str, Any]) -> Dict[str, Any]:
+    """Mark a PipeObject dict as temporary (intermediate artifact).
+
+    Args:
+            pipe_object: Result dict from cmdlet emission
+
+    Returns:
+            Modified dict with is_temp=True
+    """
+    pipe_object["is_temp"] = True
+    return pipe_object
+
+
+def set_parent_hash(pipe_object: Dict[str, Any], parent_hash: str) -> Dict[str, Any]:
+    """Set the parent_hash for provenance tracking.
+
+    Args:
+            pipe_object: Result dict
+            parent_hash: Parent file's hash
+
+    Returns:
+            Modified dict with parent_hash set to the hash
+    """
+    pipe_object["parent_hash"] = parent_hash
+    return pipe_object
+
+
+def get_pipe_object_path(pipe_object: Any) -> Optional[str]:
+    """Extract file path from PipeObject, dict, or pipeline-friendly object."""
+    if pipe_object is None:
+        return None
+    for attr in ("path", "target"):
+        if hasattr(pipe_object, attr):
+            value = getattr(pipe_object, attr)
+            if value:
+                return value
+    if isinstance(pipe_object, dict):
+        for key in ("path", "target"):
+            value = pipe_object.get(key)
+            if value:
+                return value
+    return None
+
+
+def get_pipe_object_hash(pipe_object: Any) -> Optional[str]:
+    """Extract file hash from PipeObject, dict, or pipeline-friendly object."""
+    if pipe_object is None:
+        return None
+    for attr in ("hash",):
+        if hasattr(pipe_object, attr):
+            value = getattr(pipe_object, attr)
+            if value:
+                return value
+    if isinstance(pipe_object, dict):
+        for key in ("hash",):
+            value = pipe_object.get(key)
+            if value:
+                return value
+    return None
+
+
+def resolve_item_store_hash(
+    item: Any,
+    *,
+    override_store: Optional[str] = None,
+    override_hash: Optional[str] = None,
+    hash_field: str = "hash",
+    store_field: str = "store",
+    path_fields: Sequence[str] = ("path", "target"),
+) -> Tuple[str, Optional[str]]:
+    """Resolve store name and normalized hash from a result item."""
+    store_name = str(override_store or get_field(item, store_field) or "").strip()
+    raw_hash = get_field(item, hash_field)
+
+    raw_path = None
+    for field_name in path_fields:
+        candidate = get_field(item, field_name)
+        if candidate:
+            raw_path = candidate
+            break
+
+    resolved_hash = resolve_hash_for_cmdlet(
+        str(raw_hash) if raw_hash else None,
+        str(raw_path) if raw_path else None,
+        str(override_hash) if override_hash else None,
+    )
+    return store_name, resolved_hash
+
+
+def coerce_to_pipe_object(
+    value: Any,
+    default_path: Optional[str] = None,
+) -> models.PipeObject:
+    """Normalize any incoming result to a PipeObject for single-source-of-truth state.
+
+    Uses hash+store canonical pattern.
+    """
+    if isinstance(value, models.PipeObject):
+        return value
+
+    known_keys = {
+        "hash",
+        "store",
+        "tag",
+        "title",
+        "url",
+        "source_url",
+        "duration",
+        "metadata",
+        "warnings",
+        "path",
+        "relationships",
+        "is_temp",
+        "action",
+        "parent_hash",
+    }
+
+    if hasattr(value, "to_dict"):
+        value = value.to_dict()
+    elif not isinstance(value, dict):
+        try:
+            obj_map: Dict[str, Any] = {}
+            for k in (
+                    "hash",
+                    "store",
+                    "provider",
+                    "prov",
+                    "tag",
+                    "title",
+                    "url",
+                    "source_url",
+                    "duration",
+                    "duration_seconds",
+                    "metadata",
+                    "full_metadata",
+                    "warnings",
+                    "path",
+                    "target",
+                    "relationships",
+                    "is_temp",
+                    "action",
+                    "parent_hash",
+                    "extra",
+                    "media_kind",
+            ):
+                if hasattr(value, k):
+                    obj_map[k] = getattr(value, k)
+            if obj_map:
+                value = obj_map
+        except Exception:
+            pass
+
+    if isinstance(value, dict):
+        hash_val = value.get("hash")
+        store_val = value.get("store") or "PATH"
+        if not store_val or store_val == "PATH":
+            try:
+                extra_store = value.get("extra", {}).get("store")
+            except Exception:
+                extra_store = None
+            if extra_store:
+                store_val = extra_store
+
+        if not hash_val:
+            path_val = value.get("path")
+            if path_val:
+                try:
+                    from SYS.utils import sha256_file
+
+                    hash_val = sha256_file(Path(path_val))
+                except Exception:
+                    hash_val = "unknown"
+            else:
+                hash_val = "unknown"
+
+        title_val = value.get("title")
+        if not title_val:
+            path_val = value.get("path")
+            if path_val:
+                try:
+                    title_val = Path(path_val).stem
+                except Exception:
+                    pass
+
+        extra = {
+            k: v
+            for k, v in value.items() if k not in known_keys
+        }
+
+        from SYS.metadata import normalize_urls
+
+        url_list = normalize_urls(value.get("url"))
+        url_val = url_list[0] if url_list else None
+        if len(url_list) > 1:
+            extra["url"] = url_list
+
+        rels = value.get("relationships") or {}
+
+        tag_val: list[str] = []
+        if "tag" in value:
+            raw_tag = value["tag"]
+            if isinstance(raw_tag, list):
+                tag_val = [str(t) for t in raw_tag if t is not None]
+            elif isinstance(raw_tag, str):
+                tag_val = [raw_tag]
+
+        path_val = value.get("path")
+        if not path_val and "target" in value:
+            target = value["target"]
+            if target and not (isinstance(target, str) and (target.startswith("http://")
+                                                            or target.startswith("https://"))):
+                path_val = target
+
+        try:
+            if isinstance(path_val, str) and (path_val.startswith("http://")
+                                              or path_val.startswith("https://")):
+                if not url_val:
+                    url_val = path_val
+                path_val = None
+        except Exception:
+            pass
+
+        if "media_kind" in value:
+            extra["media_kind"] = value["media_kind"]
+
+        pipe_obj = models.PipeObject(
+            hash=hash_val,
+            store=store_val,
+            plugin=str(
+                value.get("plugin")
+                or value.get("prov")
+                or value.get("source")
+                or extra.get("plugin")
+                or extra.get("source")
+                or ""
+            ).strip() or None,
+            tag=tag_val,
+            title=title_val,
+            url=url_val,
+            source_url=value.get("source_url"),
+            duration=value.get("duration") or value.get("duration_seconds"),
+            metadata=value.get("metadata") or value.get("full_metadata") or {},
+            warnings=list(value.get("warnings") or []),
+            path=path_val,
+            relationships=rels,
+            is_temp=bool(value.get("is_temp", False)),
+            action=value.get("action"),
+            parent_hash=value.get("parent_hash"),
+            extra=extra,
+        )
+
+        return pipe_obj
+
+    hash_val = "unknown"
+    path_val = default_path or getattr(value, "path", None)
+    url_val: Optional[str] = None
+    title_val = None
+
+    if isinstance(value, str):
+        s = value.strip()
+        if s.lower().startswith(("http://", "https://")):
+            url_val = s
+            path_val = None
+        else:
+            path_val = s
+
+    if path_val and path_val != "unknown":
+        try:
+            from SYS.utils import sha256_file
+
+            path_obj = Path(path_val)
+            hash_val = sha256_file(path_obj)
+            title_val = path_obj.stem
+        except Exception:
+            pass
+
+    store_val = "URL" if url_val else "PATH"
+
+    pipe_obj = models.PipeObject(
+        hash=hash_val,
+        store=store_val,
+        plugin=None,
+        path=str(path_val) if path_val and path_val != "unknown" else None,
+        title=title_val,
+        url=url_val,
+        source_url=url_val,
+        tag=[],
+        extra={},
+    )
+
+    return pipe_obj
+
+
+def propagate_metadata(
+    previous_items: Sequence[Any],
+    new_items: Sequence[Any],
+) -> List[Any]:
+    """Merge metadata/tags from previous pipeline stage into new items.
+
+    Implements "sticky metadata": items generated by a transformation (download, convert)
+    should inherit rich info (lyrics, art, tags) from their source.
+
+    Strategies:
+    A. Hash Match: If inputs/outputs share a hash, they are the same item.
+    B. Index Match: If lists are same length, assume 1:1 mapping (heuristic).
+    C. Explicit Parent: If output has `parent_hash`, link to input with that hash.
+    """
+    if not previous_items or not new_items:
+        return list(new_items)
+
+    try:
+        prev_normalized = [coerce_to_pipe_object(p) for p in previous_items]
+    except Exception:
+        return list(new_items)
+
+    prev_by_hash: Dict[str, models.PipeObject] = {}
+    for p_obj in prev_normalized:
+        if p_obj.hash and p_obj.hash != "unknown":
+            prev_by_hash[p_obj.hash] = p_obj
+
+    normalized: List[Any] = []
+
+    is_same_length = len(new_items) == len(prev_normalized)
+
+    for i, item in enumerate(new_items):
+        if isinstance(item, dict) and item.get("_skip_metadata_propagation"):
+            normalized.append(item)
+            continue
+        try:
+            obj = coerce_to_pipe_object(item)
+        except Exception:
+            normalized.append(item)
+            continue
+
+        parent: Optional[models.PipeObject] = None
+
+        if obj.hash in prev_by_hash:
+            parent = prev_by_hash[obj.hash]
+
+        if not parent and is_same_length:
+            parent = prev_normalized[i]
+
+        if not parent and obj.parent_hash and obj.parent_hash in prev_by_hash:
+            parent = prev_by_hash[obj.parent_hash]
+
+        if parent:
+            if parent.tag:
+                if not obj.tag:
+                    obj.tag = list(parent.tag)
+                else:
+                    curr_tags = {str(t).lower() for t in obj.tag}
+                    for pt in parent.tag:
+                        if str(pt).lower() not in curr_tags:
+                            obj.tag.append(pt)
+
+            if parent.metadata:
+                if not obj.metadata:
+                    obj.metadata = parent.metadata.copy()
+                else:
+                    for mk, mv in parent.metadata.items():
+                        if mk not in obj.metadata:
+                            obj.metadata[mk] = mv
+
+            if parent.source_url and not obj.source_url:
+                obj.source_url = parent.source_url
+            elif parent.url and not obj.source_url and not obj.url:
+                obj.source_url = parent.url
+
+            if parent.relationships:
+                if not obj.relationships:
+                    obj.relationships = parent.relationships.copy()
+                else:
+                    for rk, rv in parent.relationships.items():
+                        if rk not in obj.relationships:
+                            obj.relationships[rk] = rv
+
+            if parent.extra:
+                if not obj.extra:
+                    obj.extra = parent.extra.copy()
+                else:
+                    for ek, ev in parent.extra.items():
+                        if ek not in obj.extra:
+                            obj.extra[ek] = ev
+                        elif ek == "notes" and isinstance(ev, dict) and isinstance(obj.extra[ek], dict):
+                            for nk, nv in ev.items():
+                                if nk not in obj.extra[ek]:
+                                    obj.extra[ek][nk] = nv
+
+        normalized.append(obj)
+
+    return normalized
+
+
+def pipeline_item_local_path(item: Any) -> Optional[str]:
+    """Extract local file path from a pipeline item.
+
+    Supports both dataclass objects with .path attribute and dicts.
+    Returns None for HTTP/HTTPS url.
+
+    Args:
+            item: Pipeline item (PipelineItem dataclass, dict, or other)
+
+    Returns:
+            Local file path string, or None if item is not a local file
+    """
+    path_value: Optional[str] = None
+    if hasattr(item, "path"):
+        path_value = getattr(item, "path", None)
+    elif isinstance(item, dict):
+        raw = item.get("path") or item.get("url")
+        path_value = str(raw) if raw is not None else None
+    if not isinstance(path_value, str):
+        return None
+    text = path_value.strip()
+    if not text:
+        return None
+    if text.lower().startswith(("http://", "https://")):
+        return None
+    return text
+
+
+def extract_relationships(result: Any) -> Optional[Dict[str, Any]]:
+    if isinstance(result, models.PipeObject):
+        relationships = result.get_relationships()
+        return relationships or None
+    if isinstance(result, dict):
+        relationships = result.get("relationships")
+        if isinstance(relationships, dict) and relationships:
+            return relationships
+    return None
+
+
+def extract_duration(result: Any) -> Optional[float]:
+    duration = None
+    if isinstance(result, models.PipeObject):
+        duration = result.duration
+    elif isinstance(result, dict):
+        duration = result.get("duration")
+        if duration is None:
+            metadata = result.get("metadata")
+            if isinstance(metadata, dict):
+                duration = metadata.get("duration")
+    if duration is None:
+        return None
+    try:
+        return float(duration)
+    except (TypeError, ValueError):
+        return None

cmdlet/_preflight.py

@@ -0,0 +1,80 @@
+"""Store backend resolution and preflight validation helpers.
+
+Resolves store backends for cmdlets, preferring targeted instances before
+falling back to the backend registry. Used by store-related cmdlets to avoid
+repeated registry setup boilerplate.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+__all__ = [
+    "get_store_backend",
+    "get_preferred_store_backend",
+]
+
+
+def get_store_backend(
+    config: Optional[Dict[str, Any]],
+    store_name: Optional[str],
+    *,
+    store_registry: Any = None,
+    suppress_debug: bool = False,
+) -> Tuple[Optional[Any], Any, Optional[Exception]]:
+    """Resolve a store backend, optionally reusing an existing registry.
+
+    Returns a tuple of ``(backend, store_registry, exc)`` so callers can keep
+    their command-specific error messages while avoiding repeated registry setup
+    and ``store[name]`` boilerplate.
+    """
+    registry = store_registry
+    if registry is None:
+        try:
+            from PluginCore.backend_registry import BackendRegistry
+
+            registry = BackendRegistry(config or {}, suppress_debug=suppress_debug)
+        except Exception as exc:
+            return None, None, exc
+
+    backend_name = str(store_name or "").strip()
+    if not backend_name:
+        return None, registry, KeyError("Missing store name")
+
+    try:
+        return registry[backend_name], registry, None
+    except Exception as exc:
+        return None, registry, exc
+
+
+def get_preferred_store_backend(
+    config: Optional[Dict[str, Any]],
+    store_name: Optional[str],
+    *,
+    store_registry: Any = None,
+    suppress_debug: bool = True,
+) -> Tuple[Optional[Any], Any, Optional[Exception]]:
+    """Prefer a targeted backend instance before falling back to registry lookup."""
+    direct_exc: Optional[Exception] = None
+    try:
+        from PluginCore.backend_registry import get_backend_instance
+
+        backend = get_backend_instance(
+            config or {},
+            str(store_name or ""),
+            suppress_debug=suppress_debug,
+        )
+        if backend is not None:
+            return backend, store_registry, None
+    except Exception as exc:
+        direct_exc = exc
+
+    backend, registry, lookup_exc = get_store_backend(
+        config,
+        store_name,
+        store_registry=store_registry,
+        suppress_debug=suppress_debug,
+    )
+    if backend is not None:
+        return backend, registry, None
+    return None, registry, direct_exc or lookup_exc

cmdlet/_store_utils.py

@@ -0,0 +1,452 @@
+"""Store and storage operation utilities.
+
+Contains hash resolution, store batch dispatching, hash query parsing, and
+Hydrus metadata fetching helpers used by store-related cmdlets.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+
+from pathlib import Path
+
+from SYS.logger import log
+from ._preflight import get_store_backend
+
+__all__ = [
+    "resolve_hash_for_cmdlet",
+    "parse_hash_query",
+    "parse_single_hash_query",
+    "require_hash_query",
+    "require_single_hash_query",
+    "get_hash_for_operation",
+    "fetch_hydrus_metadata",
+    "coalesce_hash_value_pairs",
+    "run_store_hash_value_batches",
+    "run_store_note_batches",
+    "collect_store_hash_value_batch",
+]
+
+
+def resolve_hash_for_cmdlet(
+    raw_hash: Optional[str],
+    raw_path: Optional[str],
+    override_hash: Optional[str],
+) -> Optional[str]:
+    """Resolve a file hash for note/tag/file cmdlets.
+
+    Shared implementation used by add-note, delete-note, get-note, and similar
+    cmdlets that need to identify a file by its SHA-256 hash.
+
+    Resolution order:
+    1. ``override_hash`` — explicit hash provided via *-query* (highest priority)
+    2. ``raw_hash`` — positional hash argument
+    3. ``raw_path`` stem — if the filename stem is a 64-char hex string it is
+       treated directly as the hash (Hydrus-style naming convention)
+    4. SHA-256 computed from the file at ``raw_path``
+
+    Args:
+        raw_hash: Hash string from positional argument.
+        raw_path: Filesystem path to the file (may be None).
+        override_hash: Hash extracted from *-query* (takes precedence).
+
+    Returns:
+        Normalised 64-char lowercase hex hash, or ``None`` if unresolvable.
+    """
+    from ._tag_utils import normalize_hash
+
+    resolved = normalize_hash(override_hash) if override_hash else normalize_hash(raw_hash)
+    if resolved:
+        return resolved
+    if raw_path:
+        try:
+            p = Path(str(raw_path))
+            stem = p.stem
+            if len(stem) == 64 and all(c in "0123456789abcdef" for c in stem.lower()):
+                return stem.lower()
+            if p.exists() and p.is_file():
+                from SYS.utils import sha256_file as _sha256_file
+                return _sha256_file(p)
+        except Exception:
+            return None
+    return None
+
+
+def parse_hash_query(query: Optional[str]) -> List[str]:
+    """Parse a unified query string for `hash:` into normalized SHA256 hashes.
+
+    Supported examples:
+    - hash:<h1>
+    - hash:<h1>,<h2>,<h3>
+    - Hash: <h1> <h2> <h3>
+    - hash:{<h1>, <h2>}
+
+    Returns:
+            List of unique normalized 64-hex SHA256 hashes.
+    """
+    import re
+    from ._tag_utils import normalize_hash
+
+    q = str(query or "").strip()
+    if not q:
+        return []
+
+    m = re.match(r"^hash(?:es)?\s*:\s*(.+)$", q, flags=re.IGNORECASE)
+    if not m:
+        return []
+
+    rest = (m.group(1) or "").strip()
+    if rest.startswith("{") and rest.endswith("}"):
+        rest = rest[1:-1].strip()
+    if rest.startswith("[") and rest.endswith("]"):
+        rest = rest[1:-1].strip()
+
+    raw_parts = [p.strip() for p in re.split(r"[\s,]+", rest) if p.strip()]
+    out: List[str] = []
+    for part in raw_parts:
+        h = normalize_hash(part)
+        if not h:
+            continue
+        if h not in out:
+            out.append(h)
+    return out
+
+
+def parse_single_hash_query(query: Optional[str]) -> Optional[str]:
+    """Parse `hash:` query and require exactly one hash."""
+    hashes = parse_hash_query(query)
+    if len(hashes) != 1:
+        return None
+    return hashes[0]
+
+
+def require_hash_query(
+    query: Optional[str],
+    error_message: str,
+    *,
+    log_file: Any = None,
+) -> Tuple[List[str], bool]:
+    """Parse a multi-hash query and log a caller-provided error on invalid input."""
+    hashes = parse_hash_query(query)
+    if query and not hashes:
+        kwargs = {"file": log_file} if log_file is not None else {}
+        log(error_message, **kwargs)
+        return [], False
+    return hashes, True
+
+
+def require_single_hash_query(
+    query: Optional[str],
+    error_message: str,
+    *,
+    log_file: Any = None,
+) -> Tuple[Optional[str], bool]:
+    """Parse a single-hash query and log a caller-provided error on invalid input."""
+    query_hash = parse_single_hash_query(query)
+    if query and not query_hash:
+        kwargs = {"file": log_file} if log_file is not None else {}
+        log(error_message, **kwargs)
+        return None, False
+    return query_hash, True
+
+
+def get_hash_for_operation(
+    override_hash: Optional[str],
+    result: Any,
+    field_name: str = "hash",
+) -> Optional[str]:
+    """Get normalized hash from override or result object, consolidating common pattern.
+
+    Eliminates repeated pattern: normalize_hash(override) if override else normalize_hash(get_field(result, ...))
+
+    Args:
+            override_hash: Hash passed as command argument (takes precedence)
+            result: Object containing hash field (fallback)
+            field_name: Name of hash field in result object (default: "hash")
+
+    Returns:
+            Normalized hash string, or None if neither override nor result provides valid hash
+    """
+    from ._tag_utils import normalize_hash
+    from ._pipeobject_utils import get_field
+
+    if override_hash:
+        return normalize_hash(override_hash)
+    hash_value = (
+        get_field(result,
+                  field_name) or getattr(result,
+                                         field_name,
+                                         None) or getattr(result,
+                                                          "hash",
+                                                          None)
+    )
+    return normalize_hash(hash_value)
+
+
+def fetch_hydrus_metadata(
+    config: Any,
+    hash_hex: str,
+    *,
+    store_name: Optional[str] = None,
+    hydrus_client: Any = None,
+    **kwargs,
+) -> tuple[Optional[Dict[str, Any]], Optional[int]]:
+    """Fetch metadata from Hydrus for a given hash, consolidating common fetch pattern.
+
+    Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
+
+    Args:
+            config: Configuration object used to resolve the Hydrus plugin/store
+            hash_hex: File hash to fetch metadata for
+            store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
+            hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
+            **kwargs: Additional arguments to pass to client.fetch_file_metadata()
+                    Common: include_service_keys_to_tags, include_notes, include_file_url, include_duration, etc.
+
+    Returns:
+            Tuple of (metadata_dict, error_code)
+            - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable
+            - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute())
+    """
+    client = hydrus_client
+    hydrus_provider = None
+    try:
+        from PluginCore.registry import get_plugin
+
+        hydrus_provider = get_plugin("hydrusnetwork", config)
+    except Exception:
+        hydrus_provider = None
+
+    if client is None:
+        if hydrus_provider is not None:
+            try:
+                client = hydrus_provider.get_client(
+                    store_name=store_name if store_name else None,
+                    allow_default=not bool(store_name),
+                )
+            except Exception as exc:
+                if store_name:
+                    log(f"Hydrus client unavailable for store '{store_name}': {exc}")
+                else:
+                    log(f"Hydrus client unavailable: {exc}")
+                client = None
+        if client is None and store_name:
+            log(f"Hydrus client unavailable for store '{store_name}'")
+            return None, 1
+        if client is None and hydrus_provider is None:
+            log("Hydrus provider unavailable")
+            return None, 1
+
+    if hydrus_provider is not None:
+        try:
+            metadata = hydrus_provider.fetch_metadata(
+                hash_hex,
+                store_name=store_name if store_name else None,
+                **kwargs,
+            )
+        except Exception as exc:
+            log(f"Hydrus metadata fetch failed: {exc}")
+            return None, 1
+        if isinstance(metadata, dict):
+            return metadata, 0
+        if client is None:
+            if store_name:
+                log(f"Hydrus client unavailable for store '{store_name}'")
+            else:
+                log("Hydrus metadata unavailable")
+            return None, 1
+
+    try:
+        payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs)
+    except Exception as exc:
+        log(f"Hydrus metadata fetch failed: {exc}")
+        return None, 1
+
+    items = payload.get("metadata") if isinstance(payload, dict) else None
+    meta = items[0] if (
+        isinstance(items, list) and items and isinstance(items[0], dict)
+    ) else None
+
+    return meta, 0
+
+
+def coalesce_hash_value_pairs(
+    pairs: Sequence[Tuple[str, Sequence[str]]],
+) -> List[Tuple[str, List[str]]]:
+    """Merge duplicate hash/value pairs while preserving first-seen value order."""
+    merged: Dict[str, List[str]] = {}
+    for hash_value, values in pairs:
+        normalized_hash = str(hash_value or "").strip()
+        if not normalized_hash:
+            continue
+        bucket = merged.setdefault(normalized_hash, [])
+        seen = set(bucket)
+        for value in values or []:
+            text = str(value or "").strip()
+            if not text or text in seen:
+                continue
+            seen.add(text)
+            bucket.append(text)
+    return [(hash_value, items) for hash_value, items in merged.items() if items]
+
+
+def run_store_hash_value_batches(
+    config: Optional[Dict[str, Any]],
+    batch: Dict[str, List[Tuple[str, Sequence[str]]]],
+    *,
+    bulk_method_name: str,
+    single_method_name: str,
+    store_registry: Any = None,
+    suppress_debug: bool = False,
+    pass_config_to_bulk: bool = True,
+    pass_config_to_single: bool = True,
+) -> Tuple[Any, List[Tuple[str, int, int]]]:
+    """Dispatch grouped hash/value batches across stores.
+
+    Returns ``(store_registry, stats)`` where ``stats`` contains
+    ``(store_name, item_count, value_count)`` for each dispatched store.
+    Missing stores are skipped so callers can preserve existing warning behavior.
+    """
+    registry = store_registry
+    stats: List[Tuple[str, int, int]] = []
+    for store_name, pairs in batch.items():
+        backend, registry, _exc = get_store_backend(
+            config,
+            store_name,
+            store_registry=registry,
+            suppress_debug=suppress_debug,
+        )
+        if backend is None:
+            continue
+
+        bulk_pairs = coalesce_hash_value_pairs(pairs)
+        if not bulk_pairs:
+            continue
+
+        bulk_fn = getattr(backend, bulk_method_name, None)
+        if callable(bulk_fn):
+            if pass_config_to_bulk:
+                bulk_fn(bulk_pairs, config=config)
+            else:
+                bulk_fn(bulk_pairs)
+        else:
+            single_fn = getattr(backend, single_method_name)
+            for hash_value, values in bulk_pairs:
+                if pass_config_to_single:
+                    single_fn(hash_value, values, config=config)
+                else:
+                    single_fn(hash_value, values)
+
+        stats.append(
+            (
+                store_name,
+                len(bulk_pairs),
+                sum(len(values or []) for _hash_value, values in bulk_pairs),
+            )
+        )
+
+    return registry, stats
+
+
+def run_store_note_batches(
+    config: Optional[Dict[str, Any]],
+    batch: Dict[str, List[Tuple[str, str, str]]],
+    *,
+    store_registry: Any = None,
+    suppress_debug: bool = False,
+    on_store_error: Optional[Callable[[str, Exception], None]] = None,
+    on_unsupported_store: Optional[Callable[[str], None]] = None,
+    on_item_error: Optional[Callable[[str, str, str, Exception], None]] = None,
+) -> Tuple[Any, int]:
+    """Dispatch grouped note writes across stores while preserving item-level errors."""
+    registry = store_registry
+    success_count = 0
+    for store_name, items in batch.items():
+        backend, registry, exc = get_store_backend(
+            config,
+            store_name,
+            store_registry=registry,
+            suppress_debug=suppress_debug,
+        )
+        if backend is None:
+            if on_store_error is not None and exc is not None:
+                on_store_error(store_name, exc)
+            continue
+        supports_note_capability = getattr(backend, "supports_note_association", None)
+        if supports_note_capability is None:
+            supports_note_capability = hasattr(backend, "set_note")
+        if not bool(supports_note_capability):
+            if on_unsupported_store is not None:
+                on_unsupported_store(store_name)
+            continue
+        if not hasattr(backend, "set_note"):
+            if on_unsupported_store is not None:
+                on_unsupported_store(store_name)
+            continue
+
+        for hash_value, note_name, note_text in items:
+            try:
+                if backend.set_note(hash_value, note_name, note_text, config=config):
+                    success_count += 1
+            except Exception as item_exc:
+                if on_item_error is not None:
+                    on_item_error(store_name, hash_value, note_name, item_exc)
+
+    return registry, success_count
+
+
+def collect_store_hash_value_batch(
+    items: Sequence[Any],
+    *,
+    store_registry: Any,
+    value_resolver: Callable[[Any], Optional[Sequence[str]]],
+    override_hash: Optional[str] = None,
+    override_store: Optional[str] = None,
+    on_warning: Optional[Callable[[str], None]] = None,
+) -> Tuple[Dict[str, List[Tuple[str, List[str]]]], List[Any]]:
+    """Collect validated store/hash/value batches while preserving passthrough items."""
+    from ._tag_utils import normalize_hash
+    from ._pipeobject_utils import get_field as _get_field
+
+    batch: Dict[str, List[Tuple[str, List[str]]]] = {}
+    pass_through: List[Any] = []
+
+    for item in items:
+        pass_through.append(item)
+
+        raw_hash = override_hash or _get_field(item, "hash")
+        raw_store = override_store or _get_field(item, "store")
+        if not raw_hash or not raw_store:
+            if on_warning is not None:
+                on_warning("Item missing hash/store; skipping")
+            continue
+
+        normalized = normalize_hash(raw_hash)
+        if not normalized:
+            if on_warning is not None:
+                on_warning("Item has invalid hash; skipping")
+            continue
+
+        store_text = str(raw_store).strip()
+        if not store_text:
+            if on_warning is not None:
+                on_warning("Item has empty store; skipping")
+            continue
+
+        try:
+            is_available = bool(store_registry.is_available(store_text))
+        except Exception:
+            is_available = False
+        if not is_available:
+            if on_warning is not None:
+                on_warning(f"Store '{store_text}' not configured; skipping")
+            continue
+
+        values = [str(value).strip() for value in (value_resolver(item) or []) if str(value).strip()]
+        if not values:
+            continue
+
+        batch.setdefault(store_text, []).append((normalized, values))
+
+    return batch, pass_through
+

cmdlet/_tag_utils.py

@@ -0,0 +1,954 @@
+"""Tag-related utilities: parsing, normalization, template rendering, and extraction.
+
+Covers the full tag lifecycle:
+- Parsing raw tag arguments from command-line tokens
+- Normalizing hash strings for tag identity
+- Expanding tag group references from JSON config files
+- Rendering tag value templates with ``#(placeholder)`` and ``<transform(...)>`` syntax
+- Extracting tags, titles, and URLs from result objects (PipeObject/dict)
+- Manipulating tag lists (namespace collapse, preferred titles, relationships)
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+
+from SYS import models
+from SYS.logger import log
+from ._pipeobject_utils import merge_sequences
+
+__all__ = [
+    "set_tag_groups_path",
+    "normalize_hash",
+    "looks_like_hash",
+    "parse_tag_arguments",
+    "render_tag_value_templates",
+    "build_tag_value_lookup",
+    "_add_tag_values_to_lookup",
+    "expand_tag_groups",
+    "first_title_tag",
+    "apply_preferred_title",
+    "collapse_namespace_tags",
+    "collect_relationship_labels",
+    "extract_tag_from_result",
+    "extract_title_from_result",
+    "extract_url_from_result",
+]
+
+TAG_GROUPS_PATH: Optional[Path] = None
+
+_TAG_GROUPS_CACHE: Optional[Dict[str, List[str]]] = None
+_TAG_GROUPS_MTIME: Optional[float] = None
+
+_TAG_VALUE_TEMPLATE_RE = re.compile(r"#\(([^)]+)\)")
+_TAG_VALUE_FUNCTION_RE = re.compile(r"<([a-zA-Z_][a-zA-Z0-9_-]*)\((.*?)\)>")
+
+
+def set_tag_groups_path(path: Path) -> None:
+    """Set the path to the tag groups JSON file."""
+    global TAG_GROUPS_PATH
+    TAG_GROUPS_PATH = path
+
+
+@lru_cache(maxsize=4096)
+def _normalize_hash_cached(hash_hex: str) -> Optional[str]:
+    text = hash_hex.strip().lower()
+    if not text:
+        return None
+    if len(text) != 64:
+        return None
+    if not all(ch in "0123456789abcdef" for ch in text):
+        return None
+    return text
+
+
+def normalize_hash(hash_hex: Optional[str]) -> Optional[str]:
+    """Normalize a hash string to lowercase, or return None if invalid.
+
+    Args:
+            hash_hex: String that should be a hex hash
+
+    Returns:
+            Lowercase hash string, or None if input is not a string or is empty
+    """
+    if not isinstance(hash_hex, str):
+        return None
+    return _normalize_hash_cached(hash_hex)
+
+
+def looks_like_hash(candidate: Optional[str]) -> bool:
+    """Check if a string looks like a SHA256 hash (64 hex chars).
+
+    Args:
+            candidate: String to test
+
+    Returns:
+            True if the string is 64 lowercase hex characters
+    """
+    if not isinstance(candidate, str):
+        return False
+    text = candidate.strip().lower()
+    return len(text) == 64 and all(ch in "0123456789abcdef" for ch in text)
+
+
+def _normalize_tag_value_template_name(value: Any) -> str:
+    text = str(value or "").strip().lower()
+    if not text:
+        return ""
+    try:
+        text = re.sub(r"\s+", " ", text).strip()
+    except Exception:
+        text = " ".join(text.split())
+    return text
+
+
+def _tag_value_template_keys(value: Any) -> list[str]:
+    normalized = _normalize_tag_value_template_name(value)
+    if not normalized:
+        return []
+
+    keys = [normalized]
+
+    trimmed_hash = re.sub(r"\s*#+\s*$", "", normalized).strip()
+    if trimmed_hash and trimmed_hash not in keys:
+        keys.append(trimmed_hash)
+
+    return keys
+
+
+def _add_tag_values_to_lookup(lookup: Dict[str, List[str]], tag_text: Any) -> None:
+    text = str(tag_text or "").strip()
+    if not text or ":" not in text:
+        return
+    if _TAG_VALUE_TEMPLATE_RE.search(text) or _TAG_VALUE_FUNCTION_RE.search(text):
+        return
+
+    namespace, value = text.split(":", 1)
+    value_text = str(value or "").strip()
+    if not value_text:
+        return
+
+    for key in _tag_value_template_keys(namespace):
+        values = lookup.setdefault(key, [])
+        if value_text not in values:
+            values.append(value_text)
+
+
+def build_tag_value_lookup(
+    tags: Optional[Iterable[Any]],
+    *,
+    result: Any = None,
+) -> Dict[str, List[str]]:
+    """Build a placeholder lookup from existing tags and lightweight result fields.
+
+    Placeholder lookups use ``#(namespace)`` syntax. Namespace matching is
+    case-insensitive and trims repeated whitespace. A trailing ``#`` in the
+    placeholder is ignored so inputs like ``#(track #)`` can resolve ``track:9``.
+    """
+
+    lookup: Dict[str, List[str]] = {}
+    for tag in tags or []:
+        _add_tag_values_to_lookup(lookup, tag)
+
+    title_text = extract_title_from_result(result)
+    if title_text:
+        _add_tag_values_to_lookup(lookup, f"title:{title_text}")
+
+    return lookup
+
+
+def _split_tag_value_function_args(value: Any) -> list[str]:
+    text = str(value or "")
+    args: list[str] = []
+    current: list[str] = []
+    depth = 0
+    quote: Optional[str] = None
+    escape = False
+
+    for ch in text:
+        if escape:
+            current.append(ch)
+            escape = False
+            continue
+        if ch == "\\":
+            current.append(ch)
+            escape = True
+            continue
+        if quote:
+            current.append(ch)
+            if ch == quote:
+                quote = None
+            continue
+        if ch in {"'", '"'}:
+            current.append(ch)
+            quote = ch
+            continue
+        if ch in {"(", "[", "{"}:
+            depth += 1
+            current.append(ch)
+            continue
+        if ch in {")", "]", "}"}:
+            depth = max(0, depth - 1)
+            current.append(ch)
+            continue
+        if ch == "," and depth == 0:
+            args.append("".join(current).strip())
+            current = []
+            continue
+        current.append(ch)
+
+    tail = "".join(current).strip()
+    if tail or args:
+        args.append(tail)
+    return args
+
+
+def _strip_tag_value_function_arg(value: Any) -> str:
+    text = str(value or "").strip()
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
+        return text[1:-1]
+    return text
+
+
+def _padding_width_from_spec(value: Any) -> Optional[int]:
+    spec = _strip_tag_value_function_arg(value)
+    if not spec:
+        return None
+    if re.fullmatch(r"0+", spec):
+        return len(spec)
+    if spec.isdigit():
+        try:
+            width = int(spec)
+        except Exception:
+            return None
+        return width if width > 0 else None
+    return None
+
+
+def _replace_tag_value_placeholders(
+    value: Any,
+    lookup: Dict[str, List[str]],
+    *,
+    preserve_unresolved: bool,
+) -> tuple[str, bool]:
+    text = str(value or "")
+    unresolved = False
+
+    def _replace(match: re.Match[str]) -> str:
+        nonlocal unresolved
+        keys = _tag_value_template_keys(match.group(1) or "")
+        values: List[str] = []
+        for key in keys:
+            for candidate in lookup.get(key, []):
+                if candidate not in values:
+                    values.append(candidate)
+        if not values:
+            unresolved = True
+            return match.group(0) if preserve_unresolved else ""
+        return ", ".join(values)
+
+    return _TAG_VALUE_TEMPLATE_RE.sub(_replace, text), unresolved
+
+
+def _coerce_tag_value_integer(value: Any) -> Optional[int]:
+    text = _strip_tag_value_function_arg(value)
+    if not text:
+        return None
+    if not re.fullmatch(r"[+-]?\d+", text):
+        return None
+    try:
+        return int(text)
+    except Exception:
+        return None
+
+
+def _apply_tag_value_function(
+    name: str,
+    args: Sequence[str],
+    *,
+    lookup: Dict[str, List[str]],
+) -> Optional[str]:
+    func = str(name or "").strip().lower()
+
+    resolved_values: list[str] = []
+    unresolved_flags: list[bool] = []
+    for arg in args:
+        rendered, unresolved = _replace_tag_value_placeholders(
+            arg,
+            lookup,
+            preserve_unresolved=True,
+        )
+        resolved_values.append(_strip_tag_value_function_arg(rendered))
+        unresolved_flags.append(unresolved)
+
+    if func in {"padding", "pad", "zfill"}:
+        if len(resolved_values) != 2 or any(unresolved_flags):
+            return None
+        width = _padding_width_from_spec(resolved_values[0])
+        if width is None:
+            return None
+        return str(resolved_values[1]).zfill(width)
+
+    if func == "default":
+        if len(resolved_values) != 2:
+            return None
+        primary = resolved_values[0]
+        fallback = resolved_values[1]
+        if not unresolved_flags[0] and str(primary).strip():
+            return str(primary)
+        if unresolved_flags[1]:
+            return None
+        return str(fallback)
+
+    if func == "replace":
+        if len(resolved_values) != 3 or any(unresolved_flags):
+            return None
+        return str(resolved_values[0]).replace(
+            str(resolved_values[1]),
+            str(resolved_values[2]),
+        )
+
+    if func in {"increment", "inc", "add"}:
+        if len(resolved_values) not in {1, 2}:
+            return None
+        if unresolved_flags[0]:
+            return None
+        base_value = _coerce_tag_value_integer(resolved_values[0])
+        if base_value is None:
+            return None
+        step_value = 1
+        if len(resolved_values) == 2:
+            if unresolved_flags[1]:
+                return None
+            parsed_step = _coerce_tag_value_integer(resolved_values[1])
+            if parsed_step is None:
+                return None
+            step_value = parsed_step
+        return str(base_value + step_value)
+
+    return None
+
+
+def _render_tag_value_function_templates(
+    value: Any,
+    *,
+    lookup: Dict[str, List[str]],
+) -> tuple[str, bool]:
+    text = str(value or "")
+    unresolved = False
+
+    def _replace(match: re.Match[str]) -> str:
+        nonlocal unresolved
+        func_name = match.group(1) or ""
+        func_args = _split_tag_value_function_args(match.group(2) or "")
+        rendered = _apply_tag_value_function(
+            func_name,
+            func_args,
+            lookup=lookup,
+        )
+        if rendered is None:
+            unresolved = True
+            return match.group(0)
+        return rendered
+
+    previous = None
+    rendered = text
+    while previous != rendered and _TAG_VALUE_FUNCTION_RE.search(rendered):
+        previous = rendered
+        rendered = _TAG_VALUE_FUNCTION_RE.sub(_replace, rendered)
+        if unresolved:
+            break
+    return rendered, unresolved
+
+
+def render_tag_value_templates(
+    tags: Sequence[Any],
+    *,
+    existing_tags: Optional[Iterable[Any]] = None,
+    result: Any = None,
+) -> tuple[list[str], list[str]]:
+    """Resolve ``#(namespace)`` placeholders and ``<transform(...)>`` functions.
+
+    Returns ``(resolved_tags, unresolved_templates)``. Tags whose placeholders
+    cannot be fully resolved are omitted from ``resolved_tags`` and returned in
+    ``unresolved_templates`` so callers can warn or summarize skipped items.
+
+    Currently supported transforms:
+    - ``<padding(00,#(episode))>`` or ``<pad(2,#(episode))>`` for zero-padding
+    - ``<default(#(season),0)>`` to fall back when a placeholder is missing
+    - ``<replace(#(title),old,new)>`` for simple substring replacement
+    - ``<increment(#(episode),1)>`` for integer arithmetic
+    """
+
+    entries: list[dict[str, Any]] = []
+    lookup = build_tag_value_lookup(existing_tags, result=result)
+
+    for raw_tag in tags or []:
+        text = str(raw_tag or "").strip()
+        if not text:
+            continue
+        has_template = bool(
+            _TAG_VALUE_TEMPLATE_RE.search(text)
+            or _TAG_VALUE_FUNCTION_RE.search(text)
+        )
+        entry = {
+            "raw": text,
+            "resolved": None,
+            "has_template": has_template,
+        }
+        if not has_template:
+            entry["resolved"] = text
+            _add_tag_values_to_lookup(lookup, text)
+        entries.append(entry)
+
+    progress = True
+    while progress:
+        progress = False
+        for entry in entries:
+            if entry["resolved"] is not None or not entry["has_template"]:
+                continue
+
+            rendered, unresolved = _replace_tag_value_placeholders(
+                entry["raw"],
+                lookup,
+                preserve_unresolved=bool(_TAG_VALUE_FUNCTION_RE.search(str(entry["raw"]))),
+            )
+
+            rendered, function_unresolved = _render_tag_value_function_templates(
+                rendered,
+                lookup=lookup,
+            )
+            if function_unresolved:
+                continue
+
+            if unresolved and _TAG_VALUE_TEMPLATE_RE.search(rendered):
+                continue
+
+            rendered = rendered.strip()
+            if not rendered:
+                entry["resolved"] = ""
+                progress = True
+                continue
+
+            entry["resolved"] = rendered
+            _add_tag_values_to_lookup(lookup, rendered)
+            progress = True
+
+    resolved_tags = merge_sequences(
+        [entry["resolved"] for entry in entries if isinstance(entry.get("resolved"), str) and entry.get("resolved")],
+        case_sensitive=True,
+    )
+    unresolved_templates = [
+        str(entry["raw"])
+        for entry in entries
+        if entry["has_template"] and not entry.get("resolved")
+    ]
+    return resolved_tags, unresolved_templates
+
+
+def _normalize_tag_group_entry(value: Any) -> Optional[str]:
+    """Internal: Normalize a single tag group entry."""
+    if not isinstance(value, str):
+        value = str(value)
+    text = value.strip()
+    return text or None
+
+
+def _load_tag_groups() -> Dict[str, List[str]]:
+    """Load tag group definitions from JSON file with caching."""
+    global _TAG_GROUPS_CACHE, _TAG_GROUPS_MTIME, TAG_GROUPS_PATH
+
+    if TAG_GROUPS_PATH is None:
+        try:
+            script_dir = Path(__file__).parent.parent
+
+            candidate = script_dir / "adjective.json"
+            if candidate.exists():
+                TAG_GROUPS_PATH = candidate
+            else:
+                candidate = script_dir / "helper" / "adjective.json"
+                if candidate.exists():
+                    TAG_GROUPS_PATH = candidate
+        except Exception:
+            pass
+
+    if TAG_GROUPS_PATH is None:
+        return {}
+
+    path = TAG_GROUPS_PATH
+    try:
+        stat_result = path.stat()
+    except FileNotFoundError:
+        _TAG_GROUPS_CACHE = {}
+        _TAG_GROUPS_MTIME = None
+        return {}
+    except OSError as exc:
+        log(f"Failed to read tag groups: {exc}", file=sys.stderr)
+        _TAG_GROUPS_CACHE = {}
+        _TAG_GROUPS_MTIME = None
+        return {}
+
+    mtime = stat_result.st_mtime
+    if _TAG_GROUPS_CACHE is not None and _TAG_GROUPS_MTIME == mtime:
+        return _TAG_GROUPS_CACHE
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        log(f"Invalid tag group JSON ({path}): {exc}", file=sys.stderr)
+        _TAG_GROUPS_CACHE = {}
+        _TAG_GROUPS_MTIME = mtime
+        return {}
+
+    groups: Dict[str, List[str]] = {}
+    if isinstance(payload, dict):
+        for key, value in payload.items():
+            if not isinstance(key, str):
+                continue
+            name = key.strip().lower()
+            if not name:
+                continue
+            members: List[str] = []
+            if isinstance(value, list):
+                for entry in value:
+                    normalized = _normalize_tag_group_entry(entry)
+                    if normalized:
+                        members.append(normalized)
+            elif isinstance(value, str):
+                normalized = _normalize_tag_group_entry(value)
+                if normalized:
+                    members.extend(
+                        token.strip() for token in normalized.split(",")
+                        if token.strip()
+                    )
+            if members:
+                groups[name] = members
+
+    _TAG_GROUPS_CACHE = groups
+    _TAG_GROUPS_MTIME = mtime
+    return groups
+
+
+def expand_tag_groups(raw_tags: Iterable[str]) -> List[str]:
+    """Expand tag group references (e.g., {my_group}) into member tags.
+
+    Tag groups are defined in JSON and can be nested. Groups are referenced
+    with curly braces: {group_name}.
+
+    Args:
+            raw_tags: Sequence of tag strings, some may reference groups like "{group_name}"
+
+    Returns:
+            List of expanded tags with group references replaced
+    """
+    groups = _load_tag_groups()
+    if not groups:
+        return [tag for tag in raw_tags if isinstance(tag, str) and tag.strip()]
+
+    def _expand(tokens: Iterable[str], seen: Set[str]) -> List[str]:
+        result: List[str] = []
+        for token in tokens:
+            if not isinstance(token, str):
+                continue
+            candidate = token.strip()
+            if not candidate:
+                continue
+            if candidate.startswith("{") and candidate.endswith("}") and len(candidate) > 2:
+                name = candidate[1:-1].strip().lower()
+                if not name:
+                    continue
+                if name in seen:
+                    log(
+                        f"Tag group recursion detected for {{{name}}}; skipping",
+                        file=sys.stderr,
+                    )
+                    continue
+                members = groups.get(name)
+                if not members:
+                    log(f"Unknown tag group {{{name}}}", file=sys.stderr)
+                    result.append(candidate)
+                    continue
+                result.extend(_expand(members, seen | {name}))
+            else:
+                result.append(candidate)
+        return result
+
+    return _expand(raw_tags, set())
+
+
+def parse_tag_arguments(arguments: Sequence[str]) -> List[str]:
+    """Parse tag arguments from command line tokens.
+
+    - Supports comma-separated tags.
+    - Supports pipe namespace shorthand: "artist:A|B|C" -> artist:A, artist:B, artist:C.
+
+    Args:
+            arguments: Sequence of argument strings
+
+    Returns:
+            List of normalized tag strings (empty strings filtered out)
+    """
+
+    def _split_top_level_commas(text: str) -> List[str]:
+        segments: List[str] = []
+        current: List[str] = []
+        paren_depth = 0
+        angle_depth = 0
+        quote: Optional[str] = None
+        escape = False
+
+        for ch in text:
+            if escape:
+                current.append(ch)
+                escape = False
+                continue
+            if ch == "\\":
+                current.append(ch)
+                escape = True
+                continue
+            if quote:
+                current.append(ch)
+                if ch == quote:
+                    quote = None
+                continue
+            if ch in {"'", '"'}:
+                current.append(ch)
+                quote = ch
+                continue
+            if ch == "(":
+                paren_depth += 1
+                current.append(ch)
+                continue
+            if ch == ")":
+                paren_depth = max(0, paren_depth - 1)
+                current.append(ch)
+                continue
+            if ch == "<":
+                angle_depth += 1
+                current.append(ch)
+                continue
+            if ch == ">":
+                angle_depth = max(0, angle_depth - 1)
+                current.append(ch)
+                continue
+            if ch == "," and paren_depth == 0 and angle_depth == 0:
+                segments.append("".join(current).strip())
+                current = []
+                continue
+            current.append(ch)
+
+        tail = "".join(current).strip()
+        if tail or segments:
+            segments.append(tail)
+        return segments
+
+    def _expand_pipe_namespace(text: str) -> List[str]:
+        parts = text.split("|")
+        expanded: List[str] = []
+        last_ns: Optional[str] = None
+        for part in parts:
+            segment = part.strip()
+            if not segment:
+                continue
+            if ":" in segment:
+                ns, val = segment.split(":", 1)
+                ns = ns.strip()
+                val = val.strip()
+                last_ns = ns or last_ns
+                if last_ns and val:
+                    expanded.append(f"{last_ns}:{val}")
+                elif ns or val:
+                    expanded.append(f"{ns}:{val}".strip(":"))
+            else:
+                if last_ns:
+                    expanded.append(f"{last_ns}:{segment}")
+                else:
+                    expanded.append(segment)
+        return expanded
+
+    tags: List[str] = []
+    for argument in arguments:
+        for token in _split_top_level_commas(str(argument)):
+            text = token.strip()
+            if not text:
+                continue
+            pipe_expanded = _expand_pipe_namespace(text)
+            for entry in pipe_expanded:
+                candidate = entry.strip()
+                if not candidate:
+                    continue
+                if ":" in candidate:
+                    ns, val = candidate.split(":", 1)
+                    ns = ns.strip()
+                    val = val.strip()
+                    candidate = f"{ns}:{val}" if ns or val else ""
+                if candidate:
+                    tags.append(candidate)
+    return tags
+
+
+def first_title_tag(source: Optional[Iterable[str]]) -> Optional[str]:
+    """Find the first tag starting with "title:" in a collection.
+
+    Args:
+            source: Iterable of tag strings
+
+    Returns:
+            First title: tag found, or None
+    """
+    if not source:
+        return None
+    for item in source:
+        if not isinstance(item, str):
+            continue
+        candidate = item.strip()
+        if candidate and candidate.lower().startswith("title:"):
+            return candidate
+    return None
+
+
+def apply_preferred_title(tags: List[str], preferred: Optional[str]) -> List[str]:
+    """Replace any title: tags with a preferred title tag.
+
+    Args:
+            tags: List of tags (may contain multiple "title:" entries)
+            preferred: Preferred title tag to use (full "title: ..." format)
+
+    Returns:
+            List with old title tags removed and preferred title added (at most once)
+    """
+    if not preferred:
+        return tags
+    preferred_clean = preferred.strip()
+    if not preferred_clean:
+        return tags
+    preferred_lower = preferred_clean.lower()
+    filtered: List[str] = []
+    has_preferred = False
+    for tag in tags:
+        candidate = tag.strip()
+        if not candidate:
+            continue
+        if candidate.lower().startswith("title:"):
+            if candidate.lower() == preferred_lower:
+                if not has_preferred:
+                    filtered.append(candidate)
+                    has_preferred = True
+            continue
+        filtered.append(candidate)
+    if not has_preferred:
+        filtered.append(preferred_clean)
+    return filtered
+
+
+def collapse_namespace_tags(
+    tags: Optional[Iterable[Any]],
+    namespace: str,
+    prefer: str = "last",
+) -> list[str]:
+    """Reduce tags so only one entry for a given namespace remains.
+
+    Keeps either the first or last occurrence (default last) while preserving overall order
+    for non-matching tags. Useful for ensuring a single title: tag.
+    """
+    if not tags:
+        return []
+    ns = str(namespace or "").strip().lower()
+    if not ns:
+        return list(tags) if isinstance(tags, list) else list(tags)
+
+    prefer_last = str(prefer or "last").lower() != "first"
+    ns_prefix = ns + ":"
+
+    items = list(tags)
+    if prefer_last:
+        kept: list[str] = []
+        seen_ns = False
+        for tag in reversed(items):
+            text = str(tag)
+            if text.lower().startswith(ns_prefix):
+                if seen_ns:
+                    continue
+                seen_ns = True
+            kept.append(text)
+        kept.reverse()
+        return kept
+    else:
+        kept_ns = False
+        result: list[str] = []
+        for tag in items:
+            text = str(tag)
+            if text.lower().startswith(ns_prefix):
+                if kept_ns:
+                    continue
+                kept_ns = True
+            result.append(text)
+        return result
+
+
+def collect_relationship_labels(
+    payload: Any,
+    label_stack: List[str] | None = None,
+    mapping: Dict[str, str] | None = None,
+) -> Dict[str, str]:
+    """Recursively extract hash-to-label mappings from nested relationship data.
+
+    Walks through nested dicts/lists looking for sha256-like strings (64 hex chars)
+    and builds a mapping from hash to its path in the structure.
+
+    Example:
+            data = {
+                    "duplicates": [
+                            "abc123...",  # Will be mapped to "duplicates"
+                            {"type": "related", "items": ["def456..."]}  # Will be mapped to "duplicates / type / items"
+                    ]
+            }
+            result = collect_relationship_labels(data)
+            # result = {"abc123...": "duplicates", "def456...": "duplicates / type / items"}
+
+    Args:
+            payload: Nested data structure (dict, list, string, etc.)
+            label_stack: Internal use - tracks path during recursion
+            mapping: Internal use - accumulates hash->label mappings
+
+    Returns:
+            Dict mapping hash strings to their path labels
+    """
+    if label_stack is None:
+        label_stack = []
+    if mapping is None:
+        mapping = {}
+
+    if isinstance(payload, dict):
+        for key, value in payload.items():
+            next_stack = label_stack
+            if isinstance(key, str) and key:
+                formatted = key.replace("_", " ").strip()
+                next_stack = label_stack + [formatted]
+            collect_relationship_labels(value, next_stack, mapping)
+    elif isinstance(payload, (list, tuple, set)):
+        for value in payload:
+            collect_relationship_labels(value, label_stack, mapping)
+    elif isinstance(payload, str) and looks_like_hash(payload):
+        hash_value = payload.lower()
+        if label_stack:
+            label = " / ".join(item for item in label_stack if item)
+        else:
+            label = "related"
+        mapping.setdefault(hash_value, label)
+
+    return mapping
+
+
+def extract_tag_from_result(result: Any) -> list[str]:
+    """Extract all tags from a result dict or PipeObject.
+
+    Handles mixed types (lists, sets, strings) and various field names.
+    """
+    tag: list[str] = []
+
+    def _extend(candidate: Any) -> None:
+        if not candidate:
+            return
+        if isinstance(candidate, (list, set, tuple)):
+            tag.extend(str(t) for t in candidate if t is not None)
+        elif isinstance(candidate, str):
+            tag.append(candidate)
+
+    if isinstance(result, models.PipeObject):
+        tag.extend(result.tag or [])
+        if isinstance(result.extra, dict):
+            _extend(result.extra.get("tag"))
+        if isinstance(result.metadata, dict):
+            _extend(result.metadata.get("tag"))
+            _extend(result.metadata.get("tags"))
+    elif hasattr(result, "tag"):
+        _extend(getattr(result, "tag"))
+
+    if isinstance(result, dict):
+        _extend(result.get("tag"))
+        _extend(result.get("tags"))
+
+        extra = result.get("extra")
+        if isinstance(extra, dict):
+            _extend(extra.get("tag"))
+            _extend(extra.get("tags"))
+
+        fm = result.get("full_metadata") or result.get("metadata")
+        if isinstance(fm, dict):
+            _extend(fm.get("tag"))
+            _extend(fm.get("tags"))
+
+    return merge_sequences(tag, case_sensitive=True)
+
+
+def extract_title_from_result(result: Any) -> Optional[str]:
+    """Extract the title from a result dict or PipeObject."""
+    if isinstance(result, models.PipeObject):
+        return result.title
+    elif hasattr(result, "title"):
+        return getattr(result, "title")
+    elif isinstance(result, dict):
+        return result.get("title")
+    return None
+
+
+def extract_url_from_result(result: Any) -> list[str]:
+    """Extract all unique URLs from a result dict or PipeObject.
+
+    Handles mixed types (lists, strings) and various field names (url, source_url, webpage_url).
+    Centralizes extraction logic for cmdlets like download-file, add-file, get-url.
+    """
+    url: list[str] = []
+
+    def _extend(candidate: Any) -> None:
+        if not candidate:
+            return
+        if isinstance(candidate, list):
+            url.extend(candidate)
+        elif isinstance(candidate, str):
+            url.append(candidate)
+
+    if isinstance(result, models.PipeObject):
+        _extend(result.url)
+        _extend(result.source_url)
+        if isinstance(result.extra, dict):
+            _extend(result.extra.get("url"))
+            _extend(result.extra.get("source_url"))
+        if isinstance(result.metadata, dict):
+            _extend(result.metadata.get("url"))
+            _extend(result.metadata.get("source_url"))
+            _extend(result.metadata.get("webpage_url"))
+        if isinstance(getattr(result, "full_metadata", None), dict):
+            fm = getattr(result, "full_metadata", None)
+            if isinstance(fm, dict):
+                _extend(fm.get("url"))
+                _extend(fm.get("source_url"))
+                _extend(fm.get("webpage_url"))
+
+    elif hasattr(result, "url") or hasattr(result, "source_url"):
+        _extend(getattr(result, "url", None))
+        _extend(getattr(result, "source_url", None))
+
+    if isinstance(result, dict):
+        _extend(result.get("url"))
+        _extend(result.get("source_url"))
+        _extend(result.get("webpage_url"))
+
+        extra = result.get("extra")
+        if isinstance(extra, dict):
+            _extend(extra.get("url"))
+
+        fm = result.get("full_metadata") or result.get("metadata")
+        if isinstance(fm, dict):
+            _extend(fm.get("url"))
+            _extend(fm.get("source_url"))
+            _extend(fm.get("webpage_url"))
+
+    from SYS.metadata import normalize_urls
+    return normalize_urls(url)

cmdlet/_template_utils.py

@@ -0,0 +1,44 @@
+"""Template rendering and pipeline preview utilities.
+
+Contains helpers for constructing pipeline progress previews and any
+template-related utilities that don't belong in tag-specific modules.
+"""
+
+from __future__ import annotations
+
+from typing import Any, List, Sequence
+
+from SYS.item_accessors import get_field
+
+__all__ = [
+    "build_pipeline_preview",
+]
+
+
+def build_pipeline_preview(raw_urls: Sequence[str], piped_items: Sequence[Any]) -> List[str]:
+    """Construct a short preview list for pipeline/cmdlet progress UI."""
+    preview: List[str] = []
+
+    try:
+        for u in (raw_urls or [])[:3]:
+            if u:
+                preview.append(str(u))
+    except Exception:
+        pass
+
+    if len(preview) < 5:
+        try:
+            for item in (piped_items or [])[:5]:
+                if len(preview) >= 5:
+                    break
+                title = get_field(item, "title") or get_field(item, "target") or "Piped item"
+                preview.append(str(title))
+        except Exception:
+            pass
+
+    if not preview:
+        total = len(raw_urls or []) + len(piped_items or [])
+        if total:
+            preview.append(f"Processing {total} item(s)...")
+
+    return preview

cmdlet/_url_utils.py

@@ -0,0 +1,941 @@
+"""URL existence checks, URL merging/removal helpers, and URL preflight logic.
+
+Contains ``check_url_exists_in_storage`` with all its nested closures that
+perform bulk preflight checks across storage backends to determine whether
+URLs already exist before downloading.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from SYS.logger import log, debug, debug_panel
+from SYS import pipeline as pipeline_context
+from SYS.item_accessors import get_field
+from SYS.payload_builders import build_table_result_payload
+from SYS.result_table import Table
+from SYS.rich_display import stderr_console as get_stderr_console
+from rich.prompt import Confirm
+from contextlib import AbstractContextManager, nullcontext
+
+__all__ = [
+    "check_url_exists_in_storage",
+    "merge_urls",
+    "remove_urls",
+    "set_item_urls",
+    "register_url_with_local_library",
+]
+
+
+def register_url_with_local_library(
+    pipe_obj: Any,
+    config: Dict[str, Any],
+) -> bool:
+    """Register url with a file in the local library database.
+
+    This is called automatically by download cmdlet to ensure url are persisted
+    without requiring a separate add-url step in the pipeline.
+
+    Args:
+        pipe_obj: PipeObject with path and url
+        config: Config dict containing local library path
+
+    Returns:
+        True if url were registered, False otherwise
+    """
+    return False
+
+
+def merge_urls(existing: Any, incoming: Sequence[Any]) -> list[str]:
+    """Merge URL values into a normalized, de-duplicated list."""
+    from SYS.metadata import normalize_urls
+
+    merged: list[str] = []
+    for value in normalize_urls(existing):
+        if value not in merged:
+            merged.append(value)
+    for value in normalize_urls(list(incoming or [])):
+        if value not in merged:
+            merged.append(value)
+    return merged
+
+
+def remove_urls(existing: Any, remove: Sequence[Any]) -> list[str]:
+    """Remove URL values from an existing URL field and return survivors."""
+    from SYS.metadata import normalize_urls
+
+    current = normalize_urls(existing)
+    remove_set = {value for value in normalize_urls(list(remove or [])) if value}
+    if not remove_set:
+        return current
+    return [value for value in current if value not in remove_set]
+
+
+def set_item_urls(item: Any, urls: Sequence[Any]) -> None:
+    """Persist normalized URL values back onto a dict/object result item."""
+    normalized = merge_urls([], list(urls or []))
+    payload: Any = normalized[0] if len(normalized) == 1 else list(normalized)
+
+    try:
+        if isinstance(item, dict):
+            item["url"] = payload
+            return
+        if hasattr(item, "url"):
+            setattr(item, "url", payload)
+    except Exception:
+        return
+
+
+def check_url_exists_in_storage(
+    urls: Sequence[str],
+    storage: Any,
+    hydrus_available: bool,
+    final_output_dir: Optional[Path] = None,
+    *,
+    auto_continue_duplicates: bool = True,
+    force_prompt_in_pipeline: bool = False,
+) -> bool:
+    """Pre-flight check to see if URLs already exist in storage.
+
+    Args:
+        urls: List of URLs to check
+        storage: The storage interface
+        hydrus_available: Whether Hydrus is available
+        final_output_dir: Final output directory (to skip if same as storage)
+
+    Returns:
+        True if check passed (user said yes or no dups), False if user said no (stop).
+    """
+    if storage is None:
+        debug("Bulk URL preflight skipped: storage unavailable")
+        return True
+
+    try:
+        current_cmd_text = pipeline_context.get_current_command_text("")
+    except Exception:
+        current_cmd_text = ""
+
+    try:
+        stage_ctx = pipeline_context.get_stage_context()
+    except Exception:
+        stage_ctx = None
+
+    in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or "")))
+    start_time = time.monotonic()
+    time_budget = 45.0
+    if in_pipeline:
+        try:
+            already_checked = bool(
+                pipeline_context.load_value(
+                    "preflight.url_duplicates.checked", default=False
+                )
+            )
+        except Exception:
+            already_checked = False
+
+        if already_checked:
+            debug("Bulk URL preflight: already checked in pipeline; skipping duplicate check")
+            return True
+
+    def _load_preflight_cache() -> Dict[str, Any]:
+        try:
+            existing = pipeline_context.load_value("preflight", default=None)
+        except Exception:
+            existing = None
+        return existing if isinstance(existing, dict) else {}
+
+    def _store_preflight_cache(cache: Dict[str, Any]) -> None:
+        try:
+            pipeline_context.store_value("preflight", cache)
+        except Exception:
+            pass
+
+    def _mark_preflight_checked() -> None:
+        if not in_pipeline:
+            return
+        try:
+            pipeline_context.store_value("preflight.url_duplicates.checked", True)
+        except Exception:
+            pass
+        preflight_cache = _load_preflight_cache()
+        preflight_cache["url_duplicates_checked"] = True
+        url_dup_cache = preflight_cache.get("url_duplicates")
+        if not isinstance(url_dup_cache, dict):
+            url_dup_cache = {}
+        url_dup_cache["checked"] = True
+        preflight_cache["url_duplicates"] = url_dup_cache
+        _store_preflight_cache(preflight_cache)
+
+    def _timed_out(reason: str) -> bool:
+        try:
+            if (time.monotonic() - start_time) >= time_budget:
+                debug(
+                    f"Bulk URL preflight timed out after {time_budget:.0f}s ({reason}); continuing"
+                )
+                _mark_preflight_checked()
+                return True
+        except Exception:
+            return False
+        return False
+
+    if in_pipeline and auto_continue_duplicates:
+        try:
+            cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="")
+            cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None)
+        except Exception:
+            cached_cmd = ""
+            cached_decision = None
+
+        if (not force_prompt_in_pipeline) and cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""):
+            _mark_preflight_checked()
+            if bool(cached_decision):
+                return True
+            try:
+                pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
+            except Exception:
+                pass
+            return False
+
+    unique_urls: List[str] = []
+    for u in urls or []:
+        s = str(u or "").strip()
+        if s and s not in unique_urls:
+            unique_urls.append(s)
+    if len(unique_urls) == 0:
+        return True
+
+    try:
+        from SYS.metadata import normalize_urls
+    except Exception:
+        normalize_urls = None  # type: ignore[assignment]
+
+    def _httpish(value: str) -> bool:
+        try:
+            return bool(value) and (value.startswith("http://") or value.startswith("https://"))
+        except Exception:
+            return False
+
+    def _normalize_url_for_search(value: str) -> str:
+        url = str(value or "").strip()
+
+        url = url.split("#", 1)[0]
+
+        try:
+            parsed = urlparse(url)
+        except Exception:
+            parsed = None
+
+        if parsed is not None and parsed.query:
+            time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"}
+            tracking_prefixes = ("utm_",)
+            try:
+                pairs = parse_qsl(parsed.query, keep_blank_values=True)
+                filtered = []
+                for key, val in pairs:
+                    key_norm = str(key or "").lower()
+                    if key_norm in time_keys:
+                        continue
+                    if key_norm.startswith(tracking_prefixes):
+                        continue
+                    filtered.append((key, val))
+                if filtered:
+                    url = urlunparse(parsed._replace(query=urlencode(filtered, doseq=True)))
+                else:
+                    url = urlunparse(parsed._replace(query=""))
+            except Exception:
+                pass
+
+        url = re.sub(r"^[a-z][a-z0-9+.-]*://", "", url, flags=re.IGNORECASE)
+
+        url = re.sub(r"^www\.", "", url, flags=re.IGNORECASE)
+
+        return url.lower()
+
+    def _expand_url_variants(value: str) -> List[str]:
+        if not _httpish(value):
+            return []
+
+        try:
+            parsed = urlparse(value)
+        except Exception:
+            return []
+
+        if parsed.scheme.lower() not in {"http", "https"}:
+            return []
+
+        out: List[str] = []
+
+        def _add_variant(candidate: str) -> None:
+            _maybe_add(candidate)
+            try:
+                lower = str(candidate or "").lower()
+            except Exception:
+                lower = ""
+            if lower and lower != candidate:
+                _maybe_add(lower)
+
+            try:
+                parsed_candidate = urlparse(candidate)
+            except Exception:
+                parsed_candidate = None
+
+            if parsed_candidate is None:
+                return
+
+            host = (parsed_candidate.hostname or "").strip().lower()
+            if host.startswith("www."):
+                host = host[4:]
+                if host:
+                    netloc = host
+                    try:
+                        if parsed_candidate.port:
+                            netloc = f"{netloc}:{parsed_candidate.port}"
+                    except Exception:
+                        pass
+                    try:
+                        if parsed_candidate.username or parsed_candidate.password:
+                            userinfo = parsed_candidate.username or ""
+                            if parsed_candidate.password:
+                                userinfo = f"{userinfo}:{parsed_candidate.password}"
+                            if userinfo:
+                                netloc = f"{userinfo}@{netloc}"
+                    except Exception:
+                        pass
+                    alt = urlunparse(parsed_candidate._replace(netloc=netloc))
+                    _maybe_add(alt)
+                    try:
+                        lower_alt = alt.lower()
+                    except Exception:
+                        lower_alt = ""
+                    if lower_alt and lower_alt != alt:
+                        _maybe_add(lower_alt)
+
+        def _maybe_add(candidate: str) -> None:
+            if not candidate or candidate == value:
+                return
+            if candidate not in out:
+                out.append(candidate)
+
+        if parsed.fragment:
+            _add_variant(urlunparse(parsed._replace(fragment="")))
+
+        time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"}
+        tracking_prefixes = ("utm_",)
+
+        try:
+            query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
+        except Exception:
+            query_pairs = []
+
+        if query_pairs or parsed.fragment:
+            filtered_pairs = []
+            removed = False
+            for key, val in query_pairs:
+                key_norm = str(key or "").lower()
+                if key_norm in time_keys:
+                    removed = True
+                    continue
+                if key_norm.startswith(tracking_prefixes):
+                    removed = True
+                    continue
+                filtered_pairs.append((key, val))
+
+            if removed:
+                new_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else ""
+                _add_variant(urlunparse(parsed._replace(query=new_query, fragment="")))
+
+        return out
+
+    def _dedupe_needles(raw_needles: Sequence[str]) -> List[str]:
+        output: List[str] = []
+        seen: set[str] = set()
+        for candidate in (raw_needles or []):
+            candidate_text = str(candidate or "").strip()
+            if not candidate_text:
+                continue
+            key = candidate_text.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            output.append(candidate_text)
+        return output
+
+    url_needles: Dict[str, List[str]] = {}
+    for u in unique_urls:
+        needles: List[str] = []
+        if normalize_urls is not None:
+            try:
+                needles.extend([n for n in (normalize_urls(u) or []) if isinstance(n, str)])
+            except Exception:
+                needles = []
+        if not needles:
+            needles = [u]
+        filtered: List[str] = []
+        for n in needles:
+            n2 = str(n or "").strip()
+            if not n2:
+                continue
+            if not _httpish(n2):
+                continue
+            if n2 not in filtered:
+                filtered.append(n2)
+        lowered: List[str] = []
+        for n2 in filtered:
+            try:
+                lower = n2.lower()
+            except Exception:
+                lower = ""
+            if lower and lower != n2 and lower not in filtered and lower not in lowered:
+                lowered.append(lower)
+        normalized: List[str] = []
+        for n2 in filtered:
+            norm = _normalize_url_for_search(n2)
+            if norm and norm not in normalized and norm not in filtered:
+                normalized.append(norm)
+        expanded: List[str] = []
+        for n2 in filtered:
+            for extra in _expand_url_variants(n2):
+                if extra not in expanded and extra not in filtered and extra not in lowered:
+                    expanded.append(extra)
+                norm_extra = _normalize_url_for_search(extra)
+                if (
+                    norm_extra
+                    and norm_extra not in normalized
+                    and norm_extra not in filtered
+                    and norm_extra not in expanded
+                    and norm_extra not in lowered
+                ):
+                    normalized.append(norm_extra)
+
+        combined = filtered + expanded + lowered + normalized
+        deduped = _dedupe_needles(combined)
+        url_needles[u] = deduped if deduped else [u]
+
+    if in_pipeline:
+        preflight_cache = _load_preflight_cache()
+        url_dup_cache = preflight_cache.get("url_duplicates")
+        if not isinstance(url_dup_cache, dict):
+            url_dup_cache = {}
+        cached_urls = url_dup_cache.get("urls")
+        cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()
+
+        if cached_set:
+            all_cached = True
+            for original_url, needles in url_needles.items():
+                original_cached = str(original_url or "") in cached_set
+                needles_cached = True
+                if original_cached:
+                    for needle in (needles or []):
+                        needle_text = str(needle or "")
+                        if not needle_text:
+                            continue
+                        if needle_text not in cached_set:
+                            needles_cached = False
+                            break
+                else:
+                    needles_cached = False
+
+                if original_cached and needles_cached:
+                    continue
+
+                all_cached = False
+                break
+
+            if all_cached:
+                debug("Bulk URL preflight: cached for pipeline; skipping duplicate check")
+                _mark_preflight_checked()
+                return True
+
+    if _timed_out("before backend scan"):
+        return True
+
+    bulk_mode = len(unique_urls) > 3
+
+    def _build_bulk_patterns(needles_map: Dict[str, List[str]], max_per_url: int = 3, max_total: int = 240) -> List[str]:
+        patterns: List[str] = []
+        for _original, needles in needles_map.items():
+            for needle in (needles or [])[:max_per_url]:
+                needle_text = str(needle or "").strip()
+                if not needle_text:
+                    continue
+                if needle_text not in patterns:
+                    patterns.append(needle_text)
+                if len(patterns) >= max_total:
+                    return patterns
+        return patterns
+
+    bulk_patterns = _build_bulk_patterns(url_needles)
+
+    def _match_normalized_url(pattern_text: str, candidate_url: str) -> bool:
+        pattern_norm = _normalize_url_for_search(pattern_text)
+        candidate_norm = _normalize_url_for_search(candidate_url)
+        if not pattern_norm or not candidate_norm:
+            return False
+        if pattern_norm == candidate_norm:
+            return True
+        return pattern_norm in candidate_norm
+
+    def _extract_urls_from_hit(
+        hit: Any,
+        backend: Any,
+        *,
+        allow_backend_lookup: bool = True,
+    ) -> List[str]:
+        url_values: List[str] = []
+        try:
+            raw_urls = get_field(hit, "known_urls") or get_field(hit, "urls") or get_field(hit, "url")
+            if isinstance(raw_urls, str) and raw_urls.strip():
+                url_values.append(raw_urls.strip())
+            elif isinstance(raw_urls, (list, tuple, set)):
+                for item in raw_urls:
+                    if isinstance(item, str) and item.strip():
+                        url_values.append(item.strip())
+        except Exception:
+            url_values = []
+
+        if url_values or not allow_backend_lookup:
+            return url_values
+
+        try:
+            file_hash = get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or ""
+        except Exception:
+            file_hash = ""
+
+        if file_hash:
+            try:
+                fetched = backend.get_url(str(file_hash))
+                if isinstance(fetched, str) and fetched.strip():
+                    url_values.append(fetched.strip())
+                elif isinstance(fetched, (list, tuple, set)):
+                    for item in fetched:
+                        if isinstance(item, str) and item.strip():
+                            url_values.append(item.strip())
+            except Exception:
+                pass
+
+        return url_values
+
+    def _build_display_row_for_hit(
+        hit: Any,
+        backend_name: str,
+        original_url: str,
+    ) -> Dict[str, Any]:
+        try:
+            from SYS.result_table import build_display_row
+            extracted = build_display_row(hit, keys=["title", "store", "hash", "ext", "size"])
+        except Exception:
+            extracted = {}
+
+        try:
+            title = extracted.get("title") or get_field(hit, "title") or get_field(hit, "name") or get_field(hit, "target") or get_field(hit, "path") or "(exists)"
+        except Exception:
+            title = "(exists)"
+
+        try:
+            file_hash = extracted.get("hash") or get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or ""
+        except Exception:
+            file_hash = ""
+
+        ext = extracted.get("ext") if isinstance(extracted, dict) else ""
+        size_val = extracted.get("size") if isinstance(extracted, dict) else None
+
+        return build_table_result_payload(
+            title=str(title),
+            columns=[
+                ("Title", str(title)),
+                ("Store", str(get_field(hit, "store") or backend_name)),
+                ("Hash", str(file_hash or "")),
+                ("Ext", str(ext or "")),
+                ("Size", size_val),
+                ("URL", original_url),
+            ],
+            store=str(get_field(hit, "store") or backend_name),
+            hash=str(file_hash or ""),
+            ext=str(ext or ""),
+            size=size_val,
+            url=original_url,
+        )
+
+    def _search_backend_url_hits(
+        backend: Any,
+        backend_name: str,
+        original_url: str,
+        needles: Sequence[str],
+    ) -> Optional[Dict[str, Any]]:
+        backend_hits: List[Dict[str, Any]] = []
+
+        for needle in (needles or [])[:5]:
+            needle_stripped = str(needle or "").strip()
+            if not needle_stripped or not _httpish(needle_stripped):
+                continue
+            try:
+                query = f"url:{needle_stripped}"
+                backend_hits = backend.search(query, limit=1, minimal=True) or []
+                if backend_hits:
+                    return _build_display_row_for_hit(backend_hits[0], backend_name, original_url)
+            except Exception:
+                continue
+
+        for needle in (needles or [])[:3]:
+            needle_text = str(needle or "").strip()
+            if not needle_text:
+                continue
+            search_needle = _normalize_url_for_search(needle_text) or needle_text
+            query = f"url:*{search_needle}*"
+            try:
+                backend_hits = backend.search(query, limit=1, minimal=True) or []
+                if backend_hits:
+                    break
+            except Exception:
+                continue
+
+        if not backend_hits:
+            return None
+
+        hit = backend_hits[0]
+        return _build_display_row_for_hit(hit, backend_name, original_url)
+
+    backend_names: List[str] = []
+    try:
+        backend_names_all = storage.list_searchable_backends()
+    except Exception:
+        backend_names_all = []
+
+    for backend_name in backend_names_all:
+        try:
+            backend = storage[backend_name]
+        except Exception:
+            continue
+
+        try:
+            if str(backend_name).strip().lower() == "temp":
+                continue
+        except Exception:
+            pass
+
+        try:
+            backend_location = getattr(backend, "_location", None)
+            if backend_location and final_output_dir:
+                backend_path = Path(str(backend_location)).expanduser().resolve()
+                temp_path = Path(str(final_output_dir)).expanduser().resolve()
+                if backend_path == temp_path:
+                    continue
+        except Exception:
+            pass
+
+        backend_names.append(backend_name)
+
+    if not backend_names:
+        debug("Bulk URL preflight skipped: no searchable backends")
+        return True
+
+    try:
+        debug_panel(
+            "URL preflight",
+            [
+                ("url_count", len(unique_urls)),
+                ("pipeline", in_pipeline),
+                ("bulk_mode", bulk_mode),
+                ("backends", ", ".join(str(name) for name in backend_names)),
+            ],
+            border_style="yellow",
+        )
+    except Exception:
+        pass
+
+    seen_pairs: set[tuple[str, str]] = set()
+    matched_urls: set[str] = set()
+    match_rows: List[Dict[str, Any]] = []
+    max_rows = 200
+
+    hydrus_provider = None
+    try:
+        from PluginCore.registry import get_plugin
+
+        hydrus_provider = get_plugin("hydrusnetwork", config)
+    except Exception:
+        hydrus_provider = None
+
+    for backend_name in backend_names:
+        if _timed_out("backend scan"):
+            return True
+        if len(match_rows) >= max_rows:
+            break
+        try:
+            backend = storage[backend_name]
+        except Exception:
+            continue
+
+        is_hydrus_backend = False
+        try:
+            is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
+        except Exception:
+            is_hydrus_backend = False
+        if not is_hydrus_backend:
+            try:
+                is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork"
+            except Exception:
+                is_hydrus_backend = False
+
+        if is_hydrus_backend:
+            if not hydrus_available:
+                debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
+
+            if _timed_out("hydrus scan"):
+                return True
+
+            for original_url, needles in url_needles.items():
+                if _timed_out("hydrus per-url scan"):
+                    return True
+                if len(match_rows) >= max_rows:
+                    break
+                if (original_url, str(backend_name)) in seen_pairs:
+                    continue
+
+                found_hash: Optional[str] = None
+                found = False
+                lookup_exact = getattr(backend, "find_hashes_by_url", None)
+                if callable(lookup_exact):
+                    for needle in [original_url, *(needles or [])][:7]:
+                        needle_text = str(needle or "").strip()
+                        if not _httpish(needle_text):
+                            continue
+                        try:
+                            exact_hashes = lookup_exact(needle_text) or []
+                        except Exception:
+                            continue
+                        if not isinstance(exact_hashes, list) or not exact_hashes:
+                            continue
+                        try:
+                            found_hash = str(exact_hashes[0] or "").strip().lower()
+                        except Exception:
+                            found_hash = None
+                        found = True
+                        break
+
+                if not found:
+                    continue
+
+                seen_pairs.add((original_url, str(backend_name)))
+                matched_urls.add(original_url)
+                display_row = build_table_result_payload(
+                    title="(exists)",
+                    columns=[
+                        ("Title", "(exists)"),
+                        ("Store", str(backend_name)),
+                        ("Hash", found_hash or ""),
+                        ("URL", original_url),
+                    ],
+                    store=str(backend_name),
+                    hash=found_hash or "",
+                    url=original_url,
+                )
+                match_rows.append(display_row)
+            continue
+
+        if bulk_mode and bulk_patterns:
+            bulk_hits: Optional[List[Any]] = None
+            bulk_limit = min(2000, max(200, len(unique_urls) * 8))
+            try:
+                bulk_hits = backend.search(
+                    "url:*",
+                    limit=bulk_limit,
+                    pattern_hint=bulk_patterns,
+                ) or []
+            except Exception:
+                try:
+                    bulk_hits = backend.search("url:*", limit=bulk_limit) or []
+                except Exception:
+                    bulk_hits = None
+
+            if bulk_hits is not None:
+                for hit in bulk_hits:
+                    if _timed_out("backend bulk scan"):
+                        return True
+                    if len(match_rows) >= max_rows:
+                        break
+                    url_values = _extract_urls_from_hit(hit, backend, allow_backend_lookup=False)
+                    if not url_values:
+                        continue
+
+                    for original_url, needles in url_needles.items():
+                        if _timed_out("backend bulk scan"):
+                            return True
+                        if len(match_rows) >= max_rows:
+                            break
+                        if (original_url, str(backend_name)) in seen_pairs:
+                            continue
+
+                        matched = False
+                        for url_value in url_values:
+                            for needle in (needles or []):
+                                if _match_normalized_url(str(needle or ""), str(url_value or "")):
+                                    matched = True
+                                    break
+                            if matched:
+                                break
+
+                        if not matched:
+                            continue
+
+                        seen_pairs.add((original_url, str(backend_name)))
+                        matched_urls.add(original_url)
+                        match_rows.append(
+                            _build_display_row_for_hit(hit, str(backend_name), original_url)
+                        )
+                continue
+
+        for original_url, needles in url_needles.items():
+            if _timed_out("backend per-url scan"):
+                return True
+            if len(match_rows) >= max_rows:
+                break
+            if (original_url, str(backend_name)) in seen_pairs:
+                continue
+
+            display_row = _search_backend_url_hits(backend, str(backend_name), original_url, needles)
+            if not display_row:
+                continue
+
+            seen_pairs.add((original_url, str(backend_name)))
+            matched_urls.add(original_url)
+            match_rows.append(display_row)
+
+    if not match_rows:
+        if in_pipeline:
+            preflight_cache = _load_preflight_cache()
+            url_dup_cache = preflight_cache.get("url_duplicates")
+            if not isinstance(url_dup_cache, dict):
+                url_dup_cache = {}
+
+            cached_urls = url_dup_cache.get("urls")
+            cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()
+
+            for original_url, needles in url_needles.items():
+                cached_set.add(original_url)
+                for needle in needles or []:
+                    cached_set.add(str(needle))
+
+            url_dup_cache["urls"] = sorted(cached_set)
+            preflight_cache["url_duplicates"] = url_dup_cache
+            _store_preflight_cache(preflight_cache)
+            _mark_preflight_checked()
+        return True
+
+    table = Table(f"URL already exists ({len(matched_urls)} url(s))", max_columns=10)
+    table._interactive(True)
+    try:
+        table._perseverance(True)
+    except Exception:
+        pass
+
+    for row in match_rows:
+        table.add_result(row)
+
+    try:
+        pipeline_context.set_last_result_table_overlay(table, match_rows)
+    except Exception:
+        pass
+
+    suspend = getattr(pipeline_context, "suspend_live_progress", None)
+    cm: AbstractContextManager[Any] = nullcontext()
+    if callable(suspend):
+        try:
+            maybe_cm = suspend()
+            if maybe_cm is not None:
+                cm = maybe_cm  # type: ignore[assignment]
+        except Exception:
+            cm = nullcontext()
+
+    auto_confirm_reason: Optional[str] = None
+    if in_pipeline and stage_ctx is not None:
+        try:
+            total_stages = int(getattr(stage_ctx, "total_stages", 0))
+        except Exception:
+            total_stages = 0
+        try:
+            is_last_stage = bool(getattr(stage_ctx, "is_last_stage", False))
+        except Exception:
+            is_last_stage = False
+        if total_stages > 1 and not is_last_stage and not force_prompt_in_pipeline:
+            auto_confirm_reason = "pipeline stage (pre-last)"
+    if auto_confirm_reason is None:
+        try:
+            stdin_interactive = bool(sys.stdin and sys.stdin.isatty())
+        except Exception:
+            stdin_interactive = False
+        if not stdin_interactive:
+            auto_confirm_reason = "non-interactive stdin"
+
+    answered_yes = True
+    auto_declined = False
+    with cm:
+        get_stderr_console().print(table)
+        setattr(table, "_rendered_by_cmdlet", True)
+        if auto_confirm_reason is None:
+            answered_yes = bool(Confirm.ask("Continue?", default=False, console=get_stderr_console()))
+        else:
+            answered_yes = bool(auto_continue_duplicates)
+            auto_declined = not answered_yes
+            if answered_yes:
+                debug(
+                    f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
+                )
+                try:
+                    log(
+                        f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
+                        file=sys.stderr,
+                    )
+                except Exception:
+                    pass
+            else:
+                debug(
+                    f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input."
+                )
+                try:
+                    log(
+                        f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.",
+                        file=sys.stderr,
+                    )
+                except Exception:
+                    pass
+
+        if in_pipeline and auto_continue_duplicates:
+            try:
+                existing = pipeline_context.load_value("preflight", default=None)
+            except Exception:
+                existing = None
+            preflight_cache: Dict[str, Any] = existing if isinstance(existing, dict) else {}
+            url_dup_cache = preflight_cache.get("url_duplicates")
+            if not isinstance(url_dup_cache, dict):
+                url_dup_cache = {}
+            url_dup_cache["command"] = str(current_cmd_text or "")
+            url_dup_cache["continue"] = bool(answered_yes)
+            cached_urls = url_dup_cache.get("urls")
+            cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()
+            for original_url, needles in url_needles.items():
+                cached_set.add(original_url)
+                for needle in needles or []:
+                    cached_set.add(str(needle))
+            url_dup_cache["urls"] = sorted(cached_set)
+            preflight_cache["url_duplicates"] = url_dup_cache
+            try:
+                pipeline_context.store_value("preflight", preflight_cache)
+            except Exception:
+                pass
+
+        if not answered_yes:
+            if in_pipeline and not auto_declined:
+                 try:
+                    pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
+                 except Exception:
+                    pass
+            _mark_preflight_checked()
+            return False
+        _mark_preflight_checked()
+    return True

cmdlet/file/add_tagging.py

@@ -0,0 +1,562 @@
+"""Tag resolution, FlorenceVision integration, and metadata extraction for add-file."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+from pathlib import Path
+import sys
+import re
+
+from SYS import models
+from SYS.logger import log, debug
+
+from .. import _shared as sh
+
+# Import from add_core (safe: add_core defines these before importing this module)
+from .add_core import (
+    Add_File,
+    _SCREENSHOT_TIME_SUFFIX_RE,
+    _REMOTE_URL_PREFIXES,
+)
+
+extract_tag_from_result = sh.extract_tag_from_result
+extract_title_from_result = sh.extract_title_from_result
+extract_url_from_result = sh.extract_url_from_result
+merge_sequences = sh.merge_sequences
+extract_relationships = sh.extract_relationships
+extract_duration = sh.extract_duration
+collapse_namespace_tags = sh.collapse_namespace_tags
+resolve_media_kind_by_extension = sh.resolve_media_kind_by_extension
+get_field = sh.get_field
+
+
+def _maybe_apply_florencevision_tags(
+    media_path: Path,
+    tags: List[str],
+    config: Dict[str, Any],
+    pipe_obj: Optional[models.PipeObject] = None,
+) -> List[str]:
+    """Optionally auto-tag images using the FlorenceVision plugin helper.
+
+    Controlled via config:
+      [plugin=florencevision]
+      enabled=true
+      strict=false
+
+    If strict=false (default), failures log a warning and return the original tags.
+    If strict=true, failures raise to abort the ingest.
+    """
+    strict = False
+    try:
+        plugin_block = (config or {}).get("plugin")
+        fv_block = plugin_block.get("florencevision") if isinstance(plugin_block, dict) else None
+        enabled = False
+        if isinstance(fv_block, dict):
+            enabled = bool(fv_block.get("enabled"))
+            strict = bool(fv_block.get("strict"))
+        if not enabled:
+            return tags
+
+        from plugins.florencevision import FlorenceVisionTool
+
+        cfg_for_tool: Dict[str, Any] = config
+        try:
+            action = str(getattr(pipe_obj, "action", "") or "") if pipe_obj is not None else ""
+            cmdlet_name = ""
+            if action.lower().startswith("cmdlet:"):
+                cmdlet_name = action.split(":", 1)[1].strip().lower()
+            if cmdlet_name in {"screen-shot", "screen_shot", "screenshot"}:
+                plugin_block2 = dict((config or {}).get("plugin") or {})
+                fv_block2 = dict(plugin_block2.get("florencevision") or {})
+                fv_block2["task"] = "ocr"
+                plugin_block2["florencevision"] = fv_block2
+                cfg_for_tool = dict(config or {})
+                cfg_for_tool["plugin"] = plugin_block2
+        except Exception:
+            cfg_for_tool = config
+
+        fv = FlorenceVisionTool(cfg_for_tool)
+        if not fv.enabled() or not fv.applicable_path(media_path):
+            return tags
+
+        auto_tags = fv.tags_for_file(media_path)
+
+        try:
+            caption_text = getattr(fv, "last_caption", None)
+            if caption_text and pipe_obj is not None:
+                if not isinstance(pipe_obj.extra, dict):
+                    pipe_obj.extra = {}
+                notes = pipe_obj.extra.get("notes")
+                if not isinstance(notes, dict):
+                    notes = {}
+                notes.setdefault("caption", caption_text)
+                pipe_obj.extra["notes"] = notes
+        except Exception:
+            pass
+
+        if not auto_tags:
+            return tags
+
+        merged = merge_sequences(tags or [], auto_tags, case_sensitive=False)
+        debug(f"[add-file] FlorenceVision added {len(auto_tags)} tag(s)")
+        return merged
+    except Exception as exc:
+        strict2 = False
+        try:
+            tool_block = (config or {}).get("tool")
+            fv_block = tool_block.get("florencevision") if isinstance(tool_block, dict) else None
+            strict2 = bool(fv_block.get("strict")) if isinstance(fv_block, dict) else False
+        except Exception:
+            strict2 = False
+
+        if strict or strict2:
+            raise
+        log(f"[add-file] Warning: FlorenceVision tagging failed: {exc}", file=sys.stderr)
+        return tags
+
+
+def _normalize_hash_candidate(value: Any) -> str:
+    text = str(value or "").strip().lower()
+    if len(text) != 64:
+        return ""
+    if any(ch not in "0123456789abcdef" for ch in text):
+        return ""
+    return text
+
+
+def _parse_relationship_tag_king_alts(
+    tag_value: str
+) -> tuple[Optional[str], List[str]]:
+    """Parse a relationship tag into (king_hash, alt_hashes).
+
+    Supported formats:
+    - New:  relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>
+    - Old:  relationship: hash(king)<KING_HASH>,hash(alt)<ALT_HASH>...
+           relationship: hash(king)KING,hash(alt)ALT
+
+    For the local DB we treat the first hash listed as the king.
+    """
+    if not isinstance(tag_value, str):
+        return None, []
+
+    raw = tag_value.strip()
+    if not raw:
+        return None, []
+
+    rhs = raw
+    if ":" in raw:
+        prefix, rest = raw.split(":", 1)
+        if prefix.strip().lower() == "relationship":
+            rhs = rest.strip()
+
+    typed = re.findall(r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?", rhs)
+    if typed:
+        king: Optional[str] = None
+        alts: List[str] = []
+        for rel_type, h in typed:
+            h_norm = str(h).strip().lower()
+            if rel_type.strip().lower() == "king":
+                king = h_norm
+            elif rel_type.strip().lower() in {"alt", "related"}:
+                alts.append(h_norm)
+        if not king:
+            all_hashes = [str(h).strip().lower() for _, h in typed]
+            king = all_hashes[0] if all_hashes else None
+            alts = [h for h in all_hashes[1:] if h]
+        seen: set[str] = set()
+        alts = [
+            h for h in alts
+            if h and len(h) == 64 and not (h in seen or seen.add(h))
+        ]
+        if king and len(king) == 64:
+            return king, [h for h in alts if h != king]
+        return None, []
+
+    hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", rhs)
+    hashes = [h.strip().lower() for h in hashes if isinstance(h, str)]
+    if not hashes:
+        return None, []
+    king = hashes[0]
+    alts = hashes[1:]
+    seen2: set[str] = set()
+    alts = [
+        h for h in alts if h and len(h) == 64 and not (h in seen2 or seen2.add(h))
+    ]
+    return king, [h for h in alts if h != king]
+
+
+def _parse_relationships_king_alts(
+    relationships: Dict[str, Any],
+) -> tuple[Optional[str], List[str]]:
+    """Parse a PipeObject.relationships dict into (king_hash, alt_hashes).
+
+    Supported shapes:
+    - {"king": [KING], "alt": [ALT1, ALT2]}
+    - {"king": KING, "alt": ALT} (strings)
+    - Also treats "related" hashes as alts for persistence purposes.
+    """
+    if not isinstance(relationships, dict) or not relationships:
+        return None, []
+
+    def _first_hash(val: Any) -> Optional[str]:
+        if isinstance(val, str):
+            h = val.strip().lower()
+            return h if len(h) == 64 else None
+        if isinstance(val, list):
+            for item in val:
+                if isinstance(item, str):
+                    h = item.strip().lower()
+                    if len(h) == 64:
+                        return h
+        return None
+
+    def _many_hashes(val: Any) -> List[str]:
+        out: List[str] = []
+        if isinstance(val, str):
+            h = val.strip().lower()
+            if len(h) == 64:
+                out.append(h)
+        elif isinstance(val, list):
+            for item in val:
+                if isinstance(item, str):
+                    h = item.strip().lower()
+                    if len(h) == 64:
+                        out.append(h)
+        return out
+
+    king = _first_hash(relationships.get("king"))
+    if not king:
+        return None, []
+
+    alts = _many_hashes(relationships.get("alt"))
+    alts.extend(_many_hashes(relationships.get("related")))
+
+    seen: set[str] = set()
+    alts = [h for h in alts if h and h != king and not (h in seen or seen.add(h))]
+    return king, alts
+
+
+def _get_url(result: Any, pipe_obj: models.PipeObject) -> List[str]:
+    """Extract valid URLs from pipe object or result dict."""
+    from SYS.metadata import normalize_urls
+
+    candidates: List[str] = []
+
+    if pipe_obj.url:
+        candidates.append(pipe_obj.url)
+    if pipe_obj.source_url:
+        candidates.append(pipe_obj.source_url)
+
+    if isinstance(pipe_obj.extra, dict):
+        u = pipe_obj.extra.get("url")
+        if isinstance(u, list):
+            candidates.extend(str(x) for x in u if x)
+        elif isinstance(u, str):
+            candidates.append(u)
+
+    raw_from_result = extract_url_from_result(result)
+    if raw_from_result:
+        candidates.extend(raw_from_result)
+
+    normalized = normalize_urls(candidates)
+    return [u for u in normalized if Add_File._is_probable_url(u)]
+
+
+def _get_relationships(result: Any, pipe_obj: models.PipeObject) -> Optional[Dict[str, Any]]:
+    try:
+        rels = pipe_obj.get_relationships()
+        if rels:
+            return rels
+    except Exception:
+        pass
+    if isinstance(result, dict) and result.get("relationships"):
+        return result.get("relationships")
+    try:
+        return extract_relationships(result)
+    except Exception:
+        return None
+
+
+def _get_duration(result: Any, pipe_obj: models.PipeObject) -> Optional[float]:
+
+    def _parse_duration(value: Any) -> Optional[float]:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)):
+            return float(value) if value > 0 else None
+        if isinstance(value, str):
+            s = value.strip()
+            if not s:
+                return None
+            try:
+                candidate = float(s)
+                return candidate if candidate > 0 else None
+            except ValueError:
+                pass
+            if ":" in s:
+                parts = [p.strip() for p in s.split(":") if p.strip()]
+                if len(parts) in {2, 3} and all(p.isdigit() for p in parts):
+                    nums = [int(p) for p in parts]
+                    if len(nums) == 2:
+                        minutes, seconds = nums
+                        return float(minutes * 60 + seconds)
+                    hours, minutes, seconds = nums
+                    return float(hours * 3600 + minutes * 60 + seconds)
+        return None
+
+    parsed = _parse_duration(getattr(pipe_obj, "duration", None))
+    if parsed is not None:
+        return parsed
+    try:
+        return _parse_duration(extract_duration(result))
+    except Exception:
+        return None
+
+
+def _get_note_text(result: Any, pipe_obj: models.PipeObject, note_name: str) -> Optional[str]:
+    """Extract a named note text from a piped item.
+
+    Supports:
+    - pipe_obj.extra["notes"][note_name]
+    - result["notes"][note_name] for dict results
+    - pipe_obj.extra[note_name] / result[note_name] as fallback
+    """
+
+    def _normalize(val: Any) -> Optional[str]:
+        if val is None:
+            return None
+        if isinstance(val, bytes):
+            try:
+                val = val.decode("utf-8", errors="ignore")
+            except Exception:
+                val = str(val)
+        if isinstance(val, str):
+            text = val.strip()
+            return text if text else None
+        try:
+            text = str(val).strip()
+            return text if text else None
+        except Exception:
+            return None
+
+    note_key = str(note_name or "").strip()
+    if not note_key:
+        return None
+
+    try:
+        if isinstance(pipe_obj.extra, dict):
+            notes_val = pipe_obj.extra.get("notes")
+            if isinstance(notes_val, dict) and note_key in notes_val:
+                return _normalize(notes_val.get(note_key))
+            if note_key in pipe_obj.extra:
+                return _normalize(pipe_obj.extra.get(note_key))
+    except Exception:
+        pass
+
+    if isinstance(result, dict):
+        try:
+            notes_val = result.get("notes")
+            if isinstance(notes_val, dict) and note_key in notes_val:
+                return _normalize(notes_val.get(note_key))
+            if note_key in result:
+                return _normalize(result.get(note_key))
+        except Exception:
+            pass
+
+    return None
+
+
+def _load_sidecar_bundle(
+    media_path: Path,
+    instance: Optional[str],
+    config: Dict[str, Any],
+) -> Tuple[Optional[Path], Optional[str], List[str], List[str]]:
+    """Load sidecar metadata (placeholder — overridden by active plugins)."""
+    return None, None, [], []
+
+
+def _resolve_file_hash(
+    result: Any,
+    media_path: Path,
+    pipe_obj: models.PipeObject,
+    fallback_hash: Optional[str],
+) -> Optional[str]:
+    from SYS.utils import sha256_file
+
+    if pipe_obj.hash and pipe_obj.hash != "unknown":
+        return pipe_obj.hash
+    if fallback_hash:
+        return fallback_hash
+
+    if isinstance(result, dict):
+        candidate = result.get("hash")
+        if candidate:
+            return str(candidate)
+
+    try:
+        return sha256_file(media_path)
+    except Exception:
+        return None
+
+
+def _resolve_media_kind(path: Path) -> str:
+    return resolve_media_kind_by_extension(path)
+
+
+def _prepare_metadata(
+    result: Any,
+    media_path: Path,
+    pipe_obj: models.PipeObject,
+    config: Dict[str, Any],
+) -> Tuple[List[str], List[str], Optional[str], Optional[str]]:
+    """
+    Prepare tags, url, and title for the file.
+    Returns (tags, url, preferred_title, file_hash)
+    """
+    tags_from_result = list(pipe_obj.tag or [])
+    if not tags_from_result:
+        try:
+            tags_from_result = list(extract_tag_from_result(result) or [])
+        except Exception:
+            tags_from_result = []
+
+    url_from_result = Add_File._get_url(result, pipe_obj)
+
+    def _has_namespace_tag(tags: Sequence[str], namespace: str) -> bool:
+        namespace_text = str(namespace or "").strip().lower()
+        if not namespace_text:
+            return False
+        prefix = f"{namespace_text}:"
+        for tag in tags or []:
+            text = str(tag or "").strip().lower()
+            if text.startswith(prefix):
+                return True
+        return False
+
+    def _extract_screenshot_time_title() -> tuple[Optional[str], Optional[str]]:
+        current_title = str(preferred_title or "").strip()
+        filename_title = str(media_path.stem or "").strip()
+        if current_title and current_title != filename_title:
+            return None, None
+        if not url_from_result:
+            return None, None
+        suffix = str(media_path.suffix or "").strip().lower()
+        if suffix not in {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif", ".mhtml"}:
+            return None, None
+        match = _SCREENSHOT_TIME_SUFFIX_RE.match(str(media_path.stem or "").strip())
+        if not match:
+            return None, None
+        title_text = str(match.group("title") or "").strip().replace("_", " ").strip()
+        label_text = str(match.group("label") or "").strip().lower()
+        if not title_text or not label_text:
+            return None, None
+        return title_text, label_text
+
+    preferred_title = pipe_obj.title
+    if not preferred_title:
+        for t in tags_from_result:
+            if str(t).strip().lower().startswith("title:"):
+                candidate = t.split(":", 1)[1].strip().replace("_", " ").strip()
+                if candidate:
+                    preferred_title = candidate
+                    break
+    if not preferred_title:
+        preferred_title = extract_title_from_result(result)
+        if preferred_title:
+            preferred_title = preferred_title.replace("_", " ").strip()
+
+    derived_screenshot_title, derived_time_tag = _extract_screenshot_time_title()
+    if derived_screenshot_title and (
+        not preferred_title or str(preferred_title or "").strip() == str(media_path.stem or "").strip()
+    ):
+        preferred_title = derived_screenshot_title
+
+    store = getattr(pipe_obj, "store", None)
+    _, sidecar_hash, sidecar_tags, sidecar_url = Add_File._load_sidecar_bundle(
+        media_path, store, config
+    )
+
+    def normalize_title_tag(tag: str) -> str:
+        if str(tag).strip().lower().startswith("title:"):
+            parts = tag.split(":", 1)
+            if len(parts) == 2:
+                value = parts[1].replace("_", " ").strip()
+                return f"title:{value}"
+        return tag
+
+    tags_from_result_no_title = [
+        t for t in tags_from_result
+        if not str(t).strip().lower().startswith("title:")
+    ]
+    sidecar_tags = collapse_namespace_tags(
+        [normalize_title_tag(t) for t in sidecar_tags],
+        "title",
+        prefer="last"
+    )
+    sidecar_tags_filtered = [
+        t for t in sidecar_tags if not str(t).strip().lower().startswith("title:")
+    ]
+
+    merged_tags = merge_sequences(
+        tags_from_result_no_title,
+        sidecar_tags_filtered,
+        case_sensitive=True
+    )
+
+    if derived_time_tag and not _has_namespace_tag(merged_tags, "time") and not _has_namespace_tag(merged_tags, "timestamp"):
+        merged_tags.append(f"time:{derived_time_tag}")
+
+    if preferred_title:
+        merged_tags.append(f"title:{preferred_title}")
+
+    merged_url = merge_sequences(url_from_result, sidecar_url, case_sensitive=False)
+    merged_url = [u for u in merged_url if Add_File._is_probable_url(u)]
+
+    file_hash = Add_File._resolve_file_hash(
+        result,
+        media_path,
+        pipe_obj,
+        sidecar_hash
+    )
+
+    relationship_tags = [
+        t for t in merged_tags
+        if isinstance(t, str) and t.strip().lower().startswith("relationship:")
+    ]
+    if relationship_tags:
+        try:
+            if (not isinstance(getattr(pipe_obj, "relationships", None), dict) or not pipe_obj.relationships):
+                king: Optional[str] = None
+                alts: List[str] = []
+                for rel_tag in relationship_tags:
+                    k, a = _parse_relationship_tag_king_alts(rel_tag)
+                    if k and not king:
+                        king = k
+                    if a:
+                        alts.extend(a)
+                if king:
+                    seen_alt: set[str] = set()
+                    alts = [
+                        h for h in alts if h and h != king and len(h) == 64
+                        and not (h in seen_alt or seen_alt.add(h))
+                    ]
+                    payload: Dict[str, Any] = {"king": [king]}
+                    if alts:
+                        payload["alt"] = alts
+                    pipe_obj.relationships = payload
+        except Exception:
+            pass
+
+    merged_tags = [
+        t for t in merged_tags if
+        not (isinstance(t, str) and t.strip().lower().startswith("relationship:"))
+    ]
+
+    pipe_obj.tag = merged_tags
+    if preferred_title and not pipe_obj.title:
+        pipe_obj.title = preferred_title
+    if file_hash and not pipe_obj.hash:
+        pipe_obj.hash = file_hash
+    if isinstance(pipe_obj.extra, dict):
+        pipe_obj.extra["url"] = merged_url
+    return merged_tags, merged_url, preferred_title, file_hash

cmdlet/file/add_validation.py

@@ -0,0 +1,611 @@
+"""File validation, path resolution, and source fetching for add-file."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, List
+from pathlib import Path
+import sys
+import tempfile
+import shutil
+from urllib.parse import urlparse
+
+from SYS import models
+from SYS import pipeline as ctx
+from SYS.logger import log, debug
+from SYS.pipeline_progress import PipelineProgress
+from SYS.utils import sha256_file, sanitize_filename
+from API.HTTP import download_direct_file
+
+from .. import _shared as sh
+
+# Import from add_core (safe: add_core defines these before importing this module)
+from .add_core import (
+    Add_File,
+    _CommandDependencies,
+    SUPPORTED_MEDIA_EXTENSIONS,
+    _REMOTE_URL_PREFIXES,
+)
+
+coerce_to_path = sh.coerce_to_path
+get_field = sh.get_field
+
+
+def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]:
+    if not instance or not backend_name:
+        return None
+    try:
+        return instance[backend_name]
+    except Exception:
+        pass
+    target = str(backend_name or "").strip().lower()
+    if not target:
+        return None
+    try:
+        for candidate in instance.list_backends():
+            if isinstance(candidate, str) and candidate.strip().lower() == target:
+                try:
+                    return instance[candidate]
+                except Exception:
+                    continue
+    except Exception:
+        pass
+    return None
+
+
+def _build_provider_filename(
+    pipe_obj: models.PipeObject,
+    fallback_hash: Optional[str] = None,
+    source_url: Optional[str] = None,
+) -> str:
+    title_candidates: List[str] = []
+    title_value = getattr(pipe_obj, "title", "")
+    if title_value:
+        title_candidates.append(str(title_value))
+
+    extra = getattr(pipe_obj, "extra", {})
+    if isinstance(extra, dict):
+        candid = extra.get("name") or extra.get("title")
+        if candid:
+            title_candidates.append(str(candid))
+
+    metadata = getattr(pipe_obj, "metadata", {})
+    if isinstance(metadata, dict):
+        meta_name = metadata.get("title") or metadata.get("name")
+        if meta_name:
+            title_candidates.append(str(meta_name))
+
+    text = ""
+    for candidate in title_candidates:
+        if candidate:
+            text = candidate.strip()
+            if text:
+                break
+
+    if not text and fallback_hash:
+        text = fallback_hash[:8]
+
+    safe_name = sanitize_filename(text or "download")
+
+    ext = ""
+    if isinstance(metadata, dict):
+        ext = metadata.get("ext") or metadata.get("extension") or ""
+    if not ext and isinstance(extra, dict):
+        ext = extra.get("ext") or ""
+    if not ext and source_url:
+        try:
+            parsed = urlparse(source_url)
+            ext = Path(parsed.path).suffix.lstrip(".")
+        except Exception:
+            ext = ""
+
+    if ext:
+        ext_text = str(ext)
+        if not ext_text.startswith("."):
+            ext_text = "." + ext_text.lstrip(".")
+        if not safe_name.lower().endswith(ext_text.lower()):
+            safe_name = f"{safe_name}{ext_text}"
+
+    return safe_name or "download"
+
+
+def _maybe_download_backend_file(
+    backend: Any,
+    file_hash: str,
+    pipe_obj: models.PipeObject,
+    *,
+    output_dir: Optional[Path] = None,
+) -> tuple[Optional[Path], Optional[Path]]:
+    """Best-effort fetch of a backend file when get_file returns a URL.
+
+    Returns (downloaded_path, temp_dir_to_cleanup).
+    """
+
+    downloader = getattr(backend, "download_to_temp", None)
+    if not callable(downloader):
+        return None, None
+
+    tmp_dir: Optional[Path] = None
+    try:
+        suffix = None
+        if pipe_obj.path:
+            try:
+                suffix = Path(pipe_obj.path).suffix
+            except Exception:
+                pass
+
+        if not suffix:
+            metadata = getattr(pipe_obj, "metadata", {})
+            if isinstance(metadata, dict):
+                suffix = metadata.get("ext")
+
+        download_root = output_dir
+        if download_root is None:
+            tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
+            download_root = tmp_dir
+        if download_root is None:
+            return None, None
+
+        import inspect
+
+        sig = inspect.signature(downloader)
+        kwargs = {"temp_root": download_root}
+        if "suffix" in sig.parameters:
+            kwargs["suffix"] = suffix
+
+        pipeline_progress = PipelineProgress(ctx)
+        transfer_label = "peer transfer"
+        try:
+            transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label
+        except Exception:
+            transfer_label = "peer transfer"
+        if "pipeline_progress" in sig.parameters:
+            kwargs["pipeline_progress"] = pipeline_progress
+        if "transfer_label" in sig.parameters:
+            kwargs["transfer_label"] = transfer_label
+        if "progress_callback" in sig.parameters:
+
+            def _cb(done, total):
+                try:
+                    total_val = int(total) if total is not None else None
+                except Exception:
+                    total_val = None
+                try:
+                    if int(done or 0) <= 0:
+                        pipeline_progress.begin_transfer(
+                            label=transfer_label,
+                            total=total_val,
+                        )
+                except Exception:
+                    pass
+                try:
+                    pipeline_progress.update_transfer(
+                        label=transfer_label,
+                        completed=int(done or 0),
+                        total=total_val,
+                    )
+                except Exception:
+                    pass
+
+            kwargs["progress_callback"] = _cb
+
+        downloaded = downloader(str(file_hash), **kwargs)
+
+        if isinstance(downloaded, Path) and downloaded.exists():
+            if output_dir is not None:
+                pipe_obj.is_temp = False
+                if isinstance(pipe_obj.extra, dict):
+                    pipe_obj.extra["_direct_export_download"] = True
+                else:
+                    pipe_obj.extra = {"_direct_export_download": True}
+                return downloaded, None
+            pipe_obj.is_temp = True
+            return downloaded, tmp_dir
+    except Exception:
+        pass
+
+    if tmp_dir is not None:
+        try:
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+        except Exception:
+            pass
+    return None, None
+
+
+def _download_remote_backend_url(
+    remote_url: str,
+    pipe_obj: models.PipeObject,
+    *,
+    file_hash: Optional[str] = None,
+    output_dir: Optional[Path] = None,
+) -> tuple[Optional[Path], Optional[Path]]:
+    """Best-effort fetch of a remote backend URL.
+
+    Returns (downloaded_path, temp_dir_to_cleanup).
+    When ``output_dir`` is provided, the file is downloaded directly there and no
+    temp cleanup path is returned.
+    """
+
+    url_text = str(remote_url or "").strip()
+    if not url_text:
+        return None, None
+    if not url_text.lower().startswith(_REMOTE_URL_PREFIXES):
+        return None, None
+    if not url_text.lower().startswith(("http://", "https://")):
+        return None, None
+
+    tmp_dir: Optional[Path] = None
+    try:
+        download_root = output_dir
+        if download_root is None:
+            tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
+            download_root = tmp_dir
+
+        suggested_name = _build_provider_filename(
+            pipe_obj,
+            fallback_hash=file_hash,
+            source_url=url_text,
+        )
+        pipeline_progress = PipelineProgress(ctx)
+        try:
+            destination_label = str(download_root) if download_root is not None else "temporary workspace"
+            pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}")
+        except Exception:
+            pass
+
+        downloaded = download_direct_file(
+            url_text,
+            download_root,
+            quiet=False,
+            suggested_filename=suggested_name,
+            pipeline_progress=pipeline_progress,
+        )
+        downloaded_path = getattr(downloaded, "path", None)
+        if isinstance(downloaded_path, Path) and downloaded_path.exists():
+            if output_dir is not None:
+                pipe_obj.is_temp = False
+                if isinstance(pipe_obj.extra, dict):
+                    pipe_obj.extra["_direct_export_download"] = True
+                else:
+                    pipe_obj.extra = {"_direct_export_download": True}
+                return downloaded_path, None
+
+            pipe_obj.is_temp = True
+            return downloaded_path, tmp_dir
+    except Exception:
+        pass
+    finally:
+        try:
+            PipelineProgress(ctx).clear_status()
+        except Exception:
+            pass
+
+    if tmp_dir is not None:
+        try:
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+        except Exception:
+            pass
+    return None, None
+
+
+def _maybe_download_plugin_result(
+    result: Any,
+    pipe_obj: models.PipeObject,
+    config: Dict[str, Any],
+    deps: Optional[_CommandDependencies] = None,
+) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
+    plugin_key = None
+    for source in (
+        pipe_obj.plugin,
+        get_field(result, "plugin"),
+        get_field(result, "table"),
+    ):
+        candidate = Add_File._normalize_provider_key(source)
+        if candidate:
+            plugin_key = candidate
+            break
+
+    if not plugin_key:
+        return None, None, None
+
+    if deps is None:
+        deps = _CommandDependencies(config)
+
+    plugin = deps.get_plugin(plugin_key)
+    if plugin is None:
+        return None, None, None
+
+    try:
+        return plugin.resolve_pipe_result_download(result, pipe_obj)
+    except Exception as exc:
+        debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}")
+
+    return None, None, None
+
+
+def _download_piped_source(
+    pipe_obj: models.PipeObject,
+    config: Dict[str, Any],
+    store_instance: Optional[Any],
+    deps: Optional[_CommandDependencies] = None,
+) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
+    r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
+    r_store = str(getattr(pipe_obj, "store", None) or "").strip()
+    if not (r_hash and r_store):
+        return None, None, None
+
+    if deps is None:
+        deps = _CommandDependencies(config)
+
+    backend_registry = store_instance or deps.get_backend_registry()
+    backend = _resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None
+    if backend is None:
+        return None, None, None
+
+    try:
+        source = backend.get_file(r_hash.lower())
+        if isinstance(source, Path) and source.exists():
+            pipe_obj.path = str(source)
+            return source, str(r_hash), None
+        if isinstance(source, str) and source.strip():
+            dl_path, tmp_dir = _maybe_download_backend_file(
+                backend, str(r_hash), pipe_obj
+            )
+            if dl_path and dl_path.exists():
+                return dl_path, str(r_hash), tmp_dir
+            source_url = str(source).strip()
+            if source_url.lower().startswith(("http://", "https://")):
+                download_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
+                try:
+                    filename = _build_provider_filename(
+                        pipe_obj,
+                        str(r_hash),
+                        source_url,
+                    )
+                    downloaded = download_direct_file(
+                        source_url,
+                        download_dir,
+                        quiet=True,
+                        suggested_filename=filename,
+                    )
+                    downloaded_path = downloaded.path
+                    if downloaded_path and downloaded_path.exists():
+                        pipe_obj.is_temp = True
+                        pipe_obj.path = str(downloaded_path)
+                        return downloaded_path, str(r_hash), download_dir
+                except Exception as exc:
+                    debug(f"[add-file] Provider download failed: {exc}")
+                try:
+                    shutil.rmtree(download_dir, ignore_errors=True)
+                except Exception:
+                    pass
+    except Exception:
+        pass
+
+    return None, None, None
+
+
+def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
+    """Scan a directory for supported media files and return list of file info dicts.
+
+    Each dict contains:
+    - path: Path object
+    - name: filename
+    - hash: sha256 hash (or None if compute_hash=False)
+    - size: file size in bytes
+    - ext: file extension
+    """
+    if not directory.exists() or not directory.is_dir():
+        return []
+
+    files_info: List[Dict[str, Any]] = []
+
+    try:
+        for item in directory.iterdir():
+            if not item.is_file():
+                continue
+
+            ext = item.suffix.lower()
+            if ext not in SUPPORTED_MEDIA_EXTENSIONS:
+                continue
+
+            file_hash = None
+            if compute_hash:
+                try:
+                    file_hash = sha256_file(item)
+                except Exception as exc:
+                    debug(f"Failed to hash {item}: {exc}")
+                    continue
+
+            try:
+                size = item.stat().st_size
+            except Exception:
+                size = 0
+
+            files_info.append(
+                {
+                    "path": item,
+                    "name": item.name,
+                    "hash": file_hash,
+                    "size": size,
+                    "ext": ext,
+                }
+            )
+    except Exception as exc:
+        debug(f"Error scanning directory {directory}: {exc}")
+
+    return files_info
+
+
+def _validate_source(media_path: Optional[Path], allow_all_extensions: bool = False) -> bool:
+    """Validate that the source file exists and is supported.
+
+    Args:
+        media_path: Path to the file to validate
+        allow_all_extensions: If True, skip file type filtering for non-backend exports.
+                             If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest.
+    """
+    if media_path is None:
+        return False
+
+    if not media_path.exists() or not media_path.is_file():
+        log(f"File not found: {media_path}")
+        return False
+
+    if not allow_all_extensions:
+        file_extension = media_path.suffix.lower()
+        if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
+            log(f"❌ Unsupported file type: {file_extension}", file=sys.stderr)
+            return False
+
+    return True
+
+
+def _is_probable_url(s: Any) -> bool:
+    """Check if a string looks like a URL/magnet/identifier (vs a tag/title)."""
+    if not isinstance(s, str):
+        return False
+    val = s.strip().lower()
+    if not val:
+        return False
+    if val.startswith(_REMOTE_URL_PREFIXES):
+        return True
+    if "://" in val:
+        return True
+    if val.startswith("hash:"):
+        return False
+    return False
+
+
+def _resolve_source(
+    result: Any,
+    source_arg: Optional[str],
+    pipe_obj: models.PipeObject,
+    config: Dict[str, Any],
+    export_destination: Optional[Path] = None,
+    store_instance: Optional[Any] = None,
+    deps: Optional[_CommandDependencies] = None,
+) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
+    """Resolve the source file path from the positional source arg or pipeline result.
+
+    Returns (media_path, file_hash, temp_dir_to_cleanup).
+    """
+    # PRIORITY 1a: Prefer an explicit local path when it exists.
+    if isinstance(result, dict):
+        r_path = result.get("path")
+        r_hash = result.get("hash")
+        if r_path:
+            try:
+                p = coerce_to_path(r_path)
+                if p.exists() and p.is_file():
+                    pipe_obj.path = str(p)
+                    return p, str(r_hash) if r_hash else None, None
+            except Exception:
+                pass
+
+    # PRIORITY 1b: Try hash+store from result (fetch from backend)
+    r_hash = get_field(result, "hash") or get_field(result, "file_hash")
+    r_store = get_field(result, "store")
+
+    if r_hash and r_store:
+        try:
+            if deps is None:
+                deps = _CommandDependencies(config)
+            backend_registry = store_instance or deps.get_backend_registry()
+
+            backend = _resolve_backend_by_name(backend_registry, r_store)
+            if backend is not None:
+                mp = backend.get_file(r_hash)
+                if isinstance(mp, Path) and mp.exists():
+                    pipe_obj.path = str(mp)
+                    return mp, str(r_hash), None
+                if isinstance(mp, str) and mp.strip():
+                    try:
+                        mp_path = Path(str(mp))
+                    except Exception:
+                        mp_path = None
+                    if mp_path is not None and mp_path.exists() and mp_path.is_file():
+                        pipe_obj.path = str(mp_path)
+                        return mp_path, str(r_hash), None
+
+                    dl_path, tmp_dir = _maybe_download_backend_file(
+                        backend,
+                        str(r_hash),
+                        pipe_obj,
+                        output_dir=export_destination,
+                    )
+                    if dl_path and dl_path.exists():
+                        pipe_obj.path = str(dl_path)
+                        return dl_path, str(r_hash), tmp_dir
+
+                    dl_path, tmp_dir = _download_remote_backend_url(
+                        str(mp),
+                        pipe_obj,
+                        file_hash=str(r_hash),
+                        output_dir=export_destination,
+                    )
+                    if dl_path and dl_path.exists():
+                        pipe_obj.path = str(dl_path)
+                        return dl_path, str(r_hash), tmp_dir
+        except Exception as exc:
+            debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}")
+
+    # PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
+    candidate: Optional[Path] = None
+
+    if source_arg:
+        candidate = Path(source_arg)
+    elif pipe_obj.path:
+        candidate = Path(pipe_obj.path)
+
+    if not candidate:
+        obj = result[0] if isinstance(result, list) and result else result
+        if obj:
+            try:
+                candidate = coerce_to_path(obj)
+            except ValueError:
+                pass
+
+    if candidate:
+        s = str(candidate).lower()
+        if s.startswith(_REMOTE_URL_PREFIXES):
+            downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result(
+                result,
+                pipe_obj,
+                config,
+                deps=deps,
+            )
+            if downloaded_path:
+                pipe_obj.path = str(downloaded_path)
+                return downloaded_path, hash_hint, tmp_dir
+
+            dl_path, tmp_dir = _download_remote_backend_url(
+                str(candidate),
+                pipe_obj,
+                file_hash=get_field(result, "hash") or get_field(result, "file_hash"),
+                output_dir=export_destination,
+            )
+            if dl_path:
+                pipe_obj.path = str(dl_path)
+                hash_hint = get_field(result, "hash") or get_field(result, "file_hash")
+                return dl_path, hash_hint, tmp_dir
+
+            log("add-file could not auto-fetch remote source. Use download-file first.", file=sys.stderr)
+            return None, None, None
+
+        pipe_obj.path = str(candidate)
+        hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None)
+        return candidate, hash_hint, None
+
+    downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result(
+        result,
+        pipe_obj,
+        config,
+        deps=deps,
+    )
+    if downloaded_path:
+        pipe_obj.path = str(downloaded_path)
+        return downloaded_path, hash_hint, tmp_dir
+
+    debug(f"No resolution path matched. result type={type(result).__name__}")
+    log("File path could not be resolved")
+    return None, None, None

cmdlet/file/download_core.py

@@ -0,0 +1,897 @@
+"""Generic file/stream downloader — core orchestration.
+
+Supports:
+- Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp)
+- Piped plugin items (uses plugin.download when available)
+- Streaming sites via yt-dlp (YouTube, Bandcamp, etc.)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence as SequenceABC
+import sys
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence
+from urllib.parse import urlparse
+from contextlib import AbstractContextManager, nullcontext
+import shutil
+import webbrowser
+
+from API.HTTP import download_direct_file
+from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
+from SYS.logger import log, debug_panel, is_debug_enabled
+from SYS.payload_builders import build_file_result_payload, build_table_result_payload
+from SYS.pipeline_progress import PipelineProgress
+from SYS.result_table import Table, build_display_row
+from SYS.rich_display import stderr_console as get_stderr_console
+from SYS import pipeline as pipeline_context
+from SYS.item_accessors import get_result_title
+from rich.prompt import Prompt
+from SYS.selection_builder import (
+    build_hash_store_selection,
+    extract_selection_fields,
+    extract_urls_from_selection_args,
+    selection_args_have_url,
+)
+from SYS.utils import sha256_file
+
+try:
+    from plugins.ytdlp import YtDlpTool  # type: ignore
+except Exception:  # pragma: no cover - optional dependency for tests/runtime wrappers
+    YtDlpTool = None  # type: ignore
+
+from .. import _shared as sh
+
+Cmdlet = sh.Cmdlet
+CmdletArg = sh.CmdletArg
+SharedArgs = sh.SharedArgs
+QueryArg = sh.QueryArg
+parse_cmdlet_args = sh.parse_cmdlet_args
+register_url_with_local_library = sh.register_url_with_local_library
+coerce_to_pipe_object = sh.coerce_to_pipe_object
+get_field = sh.get_field
+resolve_target_dir = sh.resolve_target_dir
+coerce_to_path = sh.coerce_to_path
+build_pipeline_preview = sh.build_pipeline_preview
+
+
+class Download_File(Cmdlet):
+    """Class-based download-file cmdlet - direct HTTP downloads."""
+
+    def __init__(self) -> None:
+        """Initialize download-file cmdlet."""
+        super().__init__(
+            name="download-file",
+            summary="Download files or streaming media",
+            usage=
+            "download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]",
+            alias=["dl-file",
+                   "download-http"],
+            arg=[
+                SharedArgs.URL,
+                SharedArgs.PLUGIN,
+                SharedArgs.INSTANCE,
+                SharedArgs.PATH,
+                SharedArgs.QUERY,
+                CmdletArg(
+                    name="name",
+                    type="string",
+                    description="Output filename override for store exports.",
+                ),
+                CmdletArg(
+                    name="browser",
+                    type="flag",
+                    description="Open a backend-provided browser URL instead of exporting to disk when available.",
+                ),
+                QueryArg(
+                    "clip",
+                    key="clip",
+                    aliases=["range",
+                             "section",
+                             "sections"],
+                    type="string",
+                    required=False,
+                    description=(
+                        "Clip time ranges via -query keyed fields (e.g. clip:1m-2m or clip:00:01-00:10). "
+                        "Comma-separated values supported."
+                    ),
+                    query_only=True,
+                ),
+                CmdletArg(
+                    name="item",
+                    type="string",
+                    description="Item selection for playlists/formats",
+                ),
+            ],
+            detail=[
+                "Download files directly via HTTP or streaming media via yt-dlp.",
+                "Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.",
+                "Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
+                "For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
+            ],
+            exec=self.run,
+        )
+        self.register()
+
+    def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
+        """Main execution method."""
+        try:
+            debug_panel(
+                "download-file",
+                [
+                    ("args", list(args)),
+                    ("has_piped_input", bool(result)),
+                ],
+                border_style="cyan",
+            )
+        except Exception:
+            pass
+        return self._run_impl(result, args, config)
+
+    @staticmethod
+    def _path_from_download_result(result_obj: Any) -> Path:
+        """Normalize downloader return values to a concrete filesystem path."""
+        resolved = coerce_to_path(result_obj)
+        if resolved is None:
+            raise DownloadError("Could not determine downloaded file path")
+        return resolved
+
+    @staticmethod
+    def _selection_run_label(
+        run_args: Sequence[str],
+        *,
+        extra_url_prefixes: Sequence[str] = (),
+    ) -> str:
+        try:
+            urls = extract_urls_from_selection_args(
+                run_args,
+                extra_url_prefixes=extra_url_prefixes,
+            )
+            if urls:
+                return str(urls[0])
+        except Exception:
+            pass
+
+        for arg in run_args:
+            text = str(arg or "").strip()
+            if text and not text.startswith("-"):
+                return text
+        return "item"
+
+    @staticmethod
+    def _batch_progress_state(config: Optional[Dict[str, Any]]) -> tuple[bool, int, int, str]:
+        if not isinstance(config, dict):
+            return False, 0, 0, ""
+
+        suppress_nested = bool(config.get("_download_file_suppress_nested_pipe_progress"))
+        if not suppress_nested:
+            return False, 0, 0, ""
+
+        try:
+            total = max(0, int(config.get("_download_file_batch_total") or 0))
+        except Exception:
+            total = 0
+        try:
+            index = max(0, int(config.get("_download_file_batch_index") or 0))
+        except Exception:
+            index = 0
+        try:
+            label = str(config.get("_download_file_batch_label") or "").strip()
+        except Exception:
+            label = ""
+
+        return True, total, index, label
+
+    @staticmethod
+    def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]:
+        loader = registry.get("list_selection_url_prefixes")
+        if not callable(loader):
+            return []
+        try:
+            values = loader() or []
+        except Exception:
+            return []
+        return [str(value).strip().lower() for value in values if str(value or "").strip()]
+
+    def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]:
+        if value is None:
+            return None
+        try:
+            normalized = str(value).strip()
+        except Exception:
+            return None
+        if not normalized:
+            return None
+        if "." in normalized:
+            normalized = normalized.split(".", 1)[0]
+        return normalized.lower()
+
+    def _provider_key_from_item(self, item: Any) -> Optional[str]:
+        table_hint = get_field(item, "table")
+        key = self._normalize_provider_key(table_hint)
+        if key:
+            return key
+        provider_hint = get_field(item, "plugin")
+        key = self._normalize_provider_key(provider_hint)
+        if key:
+            return key
+        return self._normalize_provider_key(get_field(item, "source"))
+
+    @staticmethod
+    def _path_looks_local(value: Any) -> bool:
+        text = str(value or "").strip()
+        if not text:
+            return False
+        if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")):
+            return False
+        if len(text) >= 2 and text[1] == ":":
+            return True
+        if text.startswith(("\\", "/", ".", "~")):
+            return True
+        return Path(text).exists()
+
+    @staticmethod
+    def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str:
+        candidates = [
+            get_result_title(result, "title", "name", "filename"),
+            get_result_title(metadata or {}, "title", "name", "filename"),
+        ]
+        for candidate in candidates:
+            if candidate is None:
+                continue
+            text = str(candidate).strip()
+            if text:
+                return text
+        return ""
+
+    @staticmethod
+    def _sanitize_export_filename(name: str) -> str:
+        allowed_chars: List[str] = []
+        for ch in str(name or ""):
+            if ch.isalnum() or ch in {"-", "_", " ", "."}:
+                allowed_chars.append(ch)
+            else:
+                allowed_chars.append(" ")
+        sanitized = " ".join("".join(allowed_chars).split())
+        return sanitized or "export"
+
+    @staticmethod
+    def _unique_export_path(path: Path) -> Path:
+        if not path.exists():
+            return path
+        stem = path.stem
+        suffix = path.suffix
+        parent = path.parent
+        counter = 1
+        while True:
+            candidate = parent / f"{stem} ({counter}){suffix}"
+            if not candidate.exists():
+                return candidate
+            counter += 1
+
+    @staticmethod
+    def _load_provider_registry() -> Dict[str, Any]:
+        """Lightweight accessor for plugin helpers without hard dependencies."""
+        try:
+            from PluginCore import registry as provider_registry  # type: ignore
+            from PluginCore.base import SearchResult  # type: ignore
+
+            return {
+                "get_plugin": getattr(provider_registry, "get_plugin", None),
+                "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
+                "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
+                "SearchResult": SearchResult,
+            }
+        except Exception:
+            return {
+                "get_plugin": None,
+                "match_plugin_name_for_url": None,
+                "list_selection_url_prefixes": None,
+                "SearchResult": None,
+            }
+
+    @classmethod
+    def _normalize_duplicate_preflight_policy(cls, value: Any) -> Optional[str]:
+        text = str(value or "").strip().lower()
+        if not text:
+            return "skip"
+        mapping = {
+            "i": "ignore",
+            "ignore": "ignore",
+            "s": "skip",
+            "skip": "skip",
+            "c": "cancel",
+            "cancel": "cancel",
+        }
+        return mapping.get(text)
+
+    def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None:
+        try:
+            stage_ctx = pipeline_context.get_stage_context()
+        except Exception:
+            stage_ctx = None
+
+        is_last_stage = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False))
+        if not is_last_stage:
+            return
+
+        try:
+            quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
+        except Exception:
+            quiet_mode = False
+        if quiet_mode:
+            return
+
+        emitted_items: List[Any] = []
+        try:
+            emitted_items = list(getattr(stage_ctx, "emits", None) or []) if stage_ctx is not None else []
+        except Exception:
+            emitted_items = []
+
+        if not emitted_items:
+            return
+
+        try:
+            live_progress = pipeline_context.get_live_progress()
+        except Exception:
+            live_progress = None
+
+        if live_progress is not None:
+            try:
+                pipe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None
+                if isinstance(pipe_idx, int):
+                    live_progress.finish_pipe(int(pipe_idx), force_complete=True)
+            except Exception:
+                pass
+            try:
+                live_progress.stop()
+            except Exception:
+                pass
+            try:
+                if hasattr(pipeline_context, "set_live_progress"):
+                    pipeline_context.set_live_progress(None)
+            except Exception:
+                pass
+
+        try:
+            subject = emitted_items[0] if len(emitted_items) == 1 else list(emitted_items)
+            from .._shared import display_and_persist_items
+            display_and_persist_items(list(emitted_items), title="Result", subject=subject)
+        except Exception:
+            pass
+
+        try:
+            if stage_ctx is not None:
+                stage_ctx.emits = []
+        except Exception:
+            pass
+
+    def _maybe_show_provider_picker(
+        self,
+        *,
+        raw_urls: Sequence[str],
+        piped_items: Sequence[Any],
+        parsed: Dict[str, Any],
+        config: Dict[str, Any],
+        registry: Dict[str, Any],
+    ) -> Optional[int]:
+        """Generic hook for providers to show a selection table (e.g. Internet Archive format picker)."""
+        total_inputs = len(raw_urls or []) + len(piped_items or [])
+        if total_inputs != 1:
+            return None
+
+        target_url = None
+        if raw_urls:
+            target_url = str(raw_urls[0])
+        elif piped_items:
+            target_url = str(get_field(piped_items[0], "path") or get_field(piped_items[0], "url") or "")
+
+        if not target_url:
+            return None
+
+        match_provider_name_for_url = registry.get("match_plugin_name_for_url")
+        get_provider = registry.get("get_plugin")
+
+        provider_name = None
+        if match_provider_name_for_url:
+            try:
+                provider_name = match_provider_name_for_url(target_url)
+            except Exception:
+                pass
+
+        if provider_name and get_provider:
+            provider = get_provider(provider_name, config)
+            if provider and hasattr(provider, "maybe_show_picker"):
+                try:
+                    quiet_mode = bool(config.get("_quiet_background_output"))
+                    res = provider.maybe_show_picker(
+                        url=target_url,
+                        item=piped_items[0] if piped_items else None,
+                        parsed=parsed,
+                        config=config,
+                        quiet_mode=quiet_mode,
+                    )
+                    if res is not None:
+                        return int(res)
+                except Exception as e:
+                    debug_panel(
+                        "download-file picker error",
+                        [
+                            ("plugin", provider_name),
+                            ("url", target_url),
+                            ("error", e),
+                        ],
+                        border_style="yellow",
+                    )
+
+        return None
+
+    def _run_impl(
+        self,
+        result: Any,
+        args: Sequence[str],
+        config: Dict[str, Any]
+    ) -> int:
+        """Main download implementation for direct HTTP files."""
+        progress = PipelineProgress(pipeline_context)
+        prev_progress = None
+        had_progress_key = False
+        try:
+            try:
+                if isinstance(config, dict):
+                    had_progress_key = "_pipeline_progress" in config
+                    prev_progress = config.get("_pipeline_progress")
+                    config["_pipeline_progress"] = progress
+            except Exception:
+                pass
+
+            parsed = parse_cmdlet_args(args, self)
+            registry = self._load_provider_registry()
+            selection_url_prefixes = self._selection_url_prefixes(registry)
+            explicit_input = parsed.get("url")
+
+            url_candidates = parsed.get("url") or [
+                a for a in parsed.get("args", [])
+                if isinstance(a, str) and (
+                    a.startswith("http") or "://" in a or ":" in a
+                    or "\U0001f9f2" in a
+                    and not a.startswith("-")
+                )
+            ]
+            from SYS.metadata import normalize_urls as normalize_url_list  # lazy: avoids Cryptodome at startup
+            raw_url = normalize_url_list(url_candidates)
+            local_source_inputs: List[str] = []
+            if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input):
+                local_source_inputs = [str(explicit_input)]
+
+            quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
+
+            piped_items = []
+            if not raw_url:
+                piped_items = sh.normalize_result_items(
+                    result,
+                    include_falsey_single=True,
+                )
+
+            if piped_items and not raw_url:
+                selection_runs: List[List[str]] = []
+                residual_items: List[Any] = []
+
+                for item in piped_items:
+                    handled = False
+                    try:
+                        normalized_args, _normalized_action, item_url = extract_selection_fields(
+                            item,
+                            extra_url_prefixes=selection_url_prefixes,
+                        )
+                        if normalized_args:
+                            if selection_args_have_url(
+                                normalized_args,
+                                extra_url_prefixes=selection_url_prefixes,
+                            ):
+                                selection_runs.append(list(normalized_args))
+                                handled = True
+                            elif item_url:
+                                selection_runs.append([str(item_url)] + list(normalized_args))
+                                handled = True
+                    except Exception as e:
+                        debug_panel(
+                            "download-file selection args failed",
+                            [("error", e)],
+                            border_style="yellow",
+                        )
+                        handled = False
+                    if not handled:
+                        residual_items.append(item)
+                if selection_runs:
+                    selection_urls: List[str] = []
+
+                    for run_args in selection_runs:
+                        for u in extract_urls_from_selection_args(
+                            run_args,
+                            extra_url_prefixes=selection_url_prefixes,
+                        ):
+                            if u not in selection_urls:
+                                selection_urls.append(u)
+
+                    original_skip_preflight = None
+                    original_timeout = None
+                    original_skip_direct = None
+                    original_batch_total = None
+                    original_batch_index = None
+                    original_batch_label = None
+                    original_suppress_nested = None
+                    try:
+                        if isinstance(config, dict):
+                            original_skip_preflight = config.get("_skip_url_preflight")
+                            original_timeout = config.get("_pipeobject_timeout_seconds")
+                            original_skip_direct = config.get("_skip_direct_on_streaming_failure")
+                            original_batch_total = config.get("_download_file_batch_total")
+                            original_batch_index = config.get("_download_file_batch_index")
+                            original_batch_label = config.get("_download_file_batch_label")
+                            original_suppress_nested = config.get("_download_file_suppress_nested_pipe_progress")
+                    except Exception:
+                        original_skip_preflight = None
+                        original_timeout = None
+                        original_batch_total = None
+                        original_batch_index = None
+                        original_batch_label = None
+                        original_suppress_nested = None
+
+                    try:
+                        if selection_urls:
+                            if isinstance(config, dict):
+                                config["_skip_url_preflight"] = True
+                                config["_skip_direct_on_streaming_failure"] = True
+
+                        if isinstance(config, dict) and config.get("_pipeobject_timeout_seconds") is None:
+                            config["_pipeobject_timeout_seconds"] = 600
+
+                        successes = 0
+                        failures = 0
+                        last_code = 0
+                        total_selection = len(selection_runs)
+                        preview_items = list(selection_urls[:5]) or [
+                            self._selection_run_label(
+                                run_args,
+                                extra_url_prefixes=selection_url_prefixes,
+                            )
+                            for run_args in selection_runs[:5]
+                        ]
+                        try:
+                            progress.ensure_local_ui(
+                                label="download-file",
+                                total_items=total_selection,
+                                items_preview=preview_items,
+                            )
+                        except Exception:
+                            pass
+                        try:
+                            progress.begin_pipe(
+                                total_items=total_selection,
+                                items_preview=preview_items,
+                            )
+                        except Exception:
+                            pass
+                        for idx, run_args in enumerate(selection_runs, 1):
+                            run_label = self._selection_run_label(
+                                run_args,
+                                extra_url_prefixes=selection_url_prefixes,
+                            )
+                            try:
+                                progress.set_status(
+                                    f"downloading {idx}/{total_selection}: {run_label}"
+                                )
+                            except Exception:
+                                pass
+                            try:
+                                if isinstance(config, dict):
+                                    config["_download_file_batch_total"] = total_selection
+                                    config["_download_file_batch_index"] = idx
+                                    config["_download_file_batch_label"] = run_label
+                                    config["_download_file_suppress_nested_pipe_progress"] = True
+                            except Exception:
+                                pass
+                            exit_code = self._run_impl(None, run_args, config)
+                            if exit_code == 0:
+                                successes += 1
+                            else:
+                                failures += 1
+                                last_code = exit_code
+
+                        piped_items = residual_items
+                        if not piped_items:
+                            if successes > 0:
+                                return 0
+                            return last_code or 1
+                    finally:
+                        try:
+                            if isinstance(config, dict):
+                                if original_skip_preflight is None:
+                                    config.pop("_skip_url_preflight", None)
+                                else:
+                                    config["_skip_url_preflight"] = original_skip_preflight
+                                if original_timeout is None:
+                                    config.pop("_pipeobject_timeout_seconds", None)
+                                else:
+                                    config["_pipeobject_timeout_seconds"] = original_timeout
+                                if original_skip_direct is None:
+                                    config.pop("_skip_direct_on_streaming_failure", None)
+                                else:
+                                    config["_skip_direct_on_streaming_failure"] = original_skip_direct
+                                if original_batch_total is None:
+                                    config.pop("_download_file_batch_total", None)
+                                else:
+                                    config["_download_file_batch_total"] = original_batch_total
+                                if original_batch_index is None:
+                                    config.pop("_download_file_batch_index", None)
+                                else:
+                                    config["_download_file_batch_index"] = original_batch_index
+                                if original_batch_label is None:
+                                    config.pop("_download_file_batch_label", None)
+                                else:
+                                    config["_download_file_batch_label"] = original_batch_label
+                                if original_suppress_nested is None:
+                                    config.pop("_download_file_suppress_nested_pipe_progress", None)
+                                else:
+                                    config["_download_file_suppress_nested_pipe_progress"] = original_suppress_nested
+                        except Exception:
+                            pass
+
+            had_piped_input = False
+            try:
+                if isinstance(result, list):
+                    had_piped_input = bool(result)
+                else:
+                    had_piped_input = bool(result)
+            except Exception:
+                had_piped_input = False
+
+            if (had_piped_input and raw_url and len(raw_url) == 1
+                    and (not parsed.get("path"))):
+                candidate = str(raw_url[0] or "").strip()
+                low = candidate.lower()
+                looks_like_url = low.startswith(
+                    ("http://", "https://", "ftp://", "magnet:", "torrent:")
+                    + tuple(selection_url_prefixes)
+                )
+                looks_like_provider = (
+                    ":" in candidate and not candidate.startswith(
+                        ("http:", "https:", "ftp:", "ftps:", "file:")
+                        + tuple(selection_url_prefixes)
+                    )
+                )
+                looks_like_windows_path = (
+                    (len(candidate) >= 2 and candidate[1] == ":")
+                    or candidate.startswith("\\\\") or candidate.startswith("\\")
+                    or candidate.endswith(("\\", "/"))
+                )
+                if (not looks_like_url) and (
+                        not looks_like_provider) and looks_like_windows_path:
+                    parsed["path"] = candidate
+                    raw_url = []
+                    piped_items = sh.normalize_result_items(result, include_falsey_single=True)
+
+            if not raw_url and not piped_items and not local_source_inputs:
+                log("No url or piped items to download", file=sys.stderr)
+                return 1
+
+            picker_result = self._maybe_show_provider_picker(
+                raw_urls=raw_url,
+                piped_items=piped_items,
+                parsed=parsed,
+                config=config,
+                registry=registry,
+            )
+            if picker_result is not None:
+                return int(picker_result)
+
+            picker_result = self._maybe_show_provider_picker(
+                raw_urls=raw_url,
+                piped_items=piped_items,
+                parsed=parsed,
+                config=config,
+                registry=registry,
+            )
+            if picker_result is not None:
+                return int(picker_result)
+
+            final_output_dir = resolve_target_dir(parsed, config)
+            if not final_output_dir:
+                return 1
+
+            try:
+                debug_panel(
+                    "download-file plan",
+                    [
+                        ("output_dir", final_output_dir),
+                        ("remaining_urls", len(raw_url)),
+                        ("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))),
+                    ],
+                    border_style="cyan",
+                )
+            except Exception:
+                pass
+
+            total_items = max(1, len(raw_url or []) + len(piped_items or []))
+            preview = build_pipeline_preview(raw_url, piped_items)
+
+            progress.ensure_local_ui(
+                label="download-file",
+                total_items=total_items,
+                items_preview=preview
+            )
+
+            raw_url, preflight_exit, skipped_dupe_count = self._preflight_explicit_url_duplicates(
+                raw_urls=raw_url,
+                config=config,
+            )
+            if preflight_exit is not None:
+                return int(preflight_exit)
+
+            downloaded_count = 0
+
+            if local_source_inputs:
+                downloaded_count += self._process_explicit_local_sources(
+                    local_sources=local_source_inputs,
+                    final_output_dir=final_output_dir,
+                    parsed=parsed,
+                    progress=progress,
+                    config=config,
+                )
+
+            storage_downloaded, piped_items, storage_exit = self._process_storage_items(
+                piped_items=piped_items,
+                parsed=parsed,
+                config=config,
+                final_output_dir=final_output_dir,
+            )
+            downloaded_count += int(storage_downloaded)
+            if storage_exit is not None:
+                return int(storage_exit)
+
+            if skipped_dupe_count and not raw_url and not piped_items:
+                return 0 if downloaded_count > 0 else 0
+
+            urls_downloaded, early_exit = self._process_explicit_urls(
+                raw_urls=raw_url,
+                final_output_dir=final_output_dir,
+                config=config,
+                quiet_mode=quiet_mode,
+                registry=registry,
+                progress=progress,
+                parsed=parsed,
+                args=args,
+                context_items=(result if isinstance(result, list) else ([result] if result else [])),
+            )
+            downloaded_count += int(urls_downloaded)
+            if early_exit is not None:
+                return int(early_exit)
+
+            provider_downloaded, magnet_submissions = self._process_provider_items(
+                piped_items=piped_items,
+                final_output_dir=final_output_dir,
+                config=config,
+                quiet_mode=quiet_mode,
+                registry=registry,
+                progress=progress,
+            )
+            downloaded_count += provider_downloaded
+
+            if downloaded_count > 0 or magnet_submissions > 0:
+                self._maybe_render_download_details(config=config)
+                return 0
+
+            return 1
+
+        except Exception as e:
+            log(f"Error in download-file: {e}", file=sys.stderr)
+            return 1
+
+        finally:
+            try:
+                if isinstance(config, dict):
+                    if had_progress_key:
+                        config["_pipeline_progress"] = prev_progress
+                    else:
+                        config.pop("_pipeline_progress", None)
+            except Exception:
+                pass
+            progress.close_local_ui(force_complete=True)
+
+
+# Late-binding: import sub-module functions and attach them as methods.
+from .download_fetch import (  # noqa: E402
+    _emit_plugin_items,
+    _consume_plugin_download_result,
+    _process_explicit_urls,
+    _expand_provider_items,
+    _process_provider_items,
+    _download_provider_items,
+    _emit_local_file,
+    _resolve_provider_preflight_items,
+    _build_provider_playlist_item_selector,
+    _format_timecode,
+    _rebase_subtitle_timestamp_text,
+    _format_clip_range,
+    _apply_clip_decorations,
+    _parse_clip_spec_to_ranges,
+)
+
+from .download_storage import (  # noqa: E402
+    _iter_storage_export_refs,
+    _export_store_file,
+    _process_storage_items,
+    _process_explicit_local_sources,
+    _extract_hash_from_search_hit,
+    _iter_duplicate_tag_values,
+    _extract_duplicate_namespace_tags,
+    _extract_duplicate_title_tag,
+    _extract_duplicate_title,
+    _has_duplicate_title,
+    _build_duplicate_display_row,
+    _fetch_duplicate_metadata_for_hash,
+    _enrich_duplicate_metadata,
+    _fetch_duplicate_metadata_for_hashes,
+    _collect_existing_url_match_refs_for_url,
+    _find_existing_url_matches_for_url,
+    _find_existing_hash_for_url,
+    _find_existing_hashes_for_url,
+    _init_storage,
+    _supports_storage_duplicate_lookup,
+    _filter_supported_urls,
+    _canonicalize_url_for_storage,
+    _preflight_url_duplicate,
+    _preflight_explicit_url_duplicates,
+    _download_supported_urls,
+    _maybe_show_playlist_table,
+    _maybe_show_format_table_for_single_url,
+    _run_streaming_urls,
+)
+
+Download_File._emit_plugin_items = _emit_plugin_items
+Download_File._consume_plugin_download_result = _consume_plugin_download_result
+Download_File._process_explicit_urls = _process_explicit_urls
+Download_File._expand_provider_items = _expand_provider_items
+Download_File._process_provider_items = _process_provider_items
+Download_File._download_provider_items = _download_provider_items
+Download_File._emit_local_file = _emit_local_file
+Download_File._resolve_provider_preflight_items = staticmethod(_resolve_provider_preflight_items)
+Download_File._build_provider_playlist_item_selector = staticmethod(_build_provider_playlist_item_selector)
+Download_File._format_timecode = staticmethod(_format_timecode)
+Download_File._rebase_subtitle_timestamp_text = staticmethod(_rebase_subtitle_timestamp_text)
+Download_File._format_clip_range = classmethod(_format_clip_range)
+Download_File._apply_clip_decorations = classmethod(_apply_clip_decorations)
+Download_File._parse_clip_spec_to_ranges = staticmethod(_parse_clip_spec_to_ranges)
+
+Download_File._iter_storage_export_refs = staticmethod(_iter_storage_export_refs)
+Download_File._export_store_file = _export_store_file
+Download_File._process_storage_items = _process_storage_items
+Download_File._process_explicit_local_sources = _process_explicit_local_sources
+Download_File._extract_hash_from_search_hit = classmethod(_extract_hash_from_search_hit)
+Download_File._iter_duplicate_tag_values = staticmethod(_iter_duplicate_tag_values)
+Download_File._extract_duplicate_namespace_tags = staticmethod(_extract_duplicate_namespace_tags)
+Download_File._extract_duplicate_title_tag = staticmethod(_extract_duplicate_title_tag)
+Download_File._extract_duplicate_title = classmethod(_extract_duplicate_title)
+Download_File._has_duplicate_title = classmethod(_has_duplicate_title)
+Download_File._build_duplicate_display_row = classmethod(_build_duplicate_display_row)
+Download_File._fetch_duplicate_metadata_for_hash = classmethod(_fetch_duplicate_metadata_for_hash)
+Download_File._enrich_duplicate_metadata = classmethod(_enrich_duplicate_metadata)
+Download_File._fetch_duplicate_metadata_for_hashes = classmethod(_fetch_duplicate_metadata_for_hashes)
+Download_File._collect_existing_url_match_refs_for_url = classmethod(_collect_existing_url_match_refs_for_url)
+Download_File._find_existing_url_matches_for_url = classmethod(_find_existing_url_matches_for_url)
+Download_File._find_existing_hash_for_url = classmethod(_find_existing_hash_for_url)
+Download_File._find_existing_hashes_for_url = classmethod(_find_existing_hashes_for_url)
+Download_File._init_storage = staticmethod(_init_storage)
+Download_File._supports_storage_duplicate_lookup = staticmethod(_supports_storage_duplicate_lookup)
+Download_File._filter_supported_urls = staticmethod(_filter_supported_urls)
+Download_File._canonicalize_url_for_storage = staticmethod(_canonicalize_url_for_storage)
+Download_File._preflight_url_duplicate = staticmethod(_preflight_url_duplicate)
+Download_File._preflight_explicit_url_duplicates = _preflight_explicit_url_duplicates
+Download_File._download_supported_urls = _download_supported_urls
+Download_File._maybe_show_playlist_table = _maybe_show_playlist_table
+Download_File._maybe_show_format_table_for_single_url = _maybe_show_format_table_for_single_url
+Download_File._run_streaming_urls = _run_streaming_urls
+
+CMDLET = Download_File()

cmdlet/file/screenshot.py

@@ -43,29 +43,17 @@ from SYS import pipeline as pipeline_context
 # Playwright & Screenshot Dependencies
 # ============================================================================
 
-from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool
+from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool, USER_AGENT
 
 try:
     from SYS.config import resolve_output_dir
 except ImportError:
-    try:
+    resolve_output_dir = None
-        _parent_dir = str(Path(__file__).parent.parent)
-        if _parent_dir not in sys.path:
-            sys.path.insert(0, _parent_dir)
-        from SYS.config import resolve_output_dir
-    except ImportError:
-        resolve_output_dir = None
 
 # ============================================================================
 # Screenshot Constants & Configuration
 # ============================================================================
 
-USER_AGENT = (
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/120.0.0.0 Safari/537.36"
-)
-
 DEFAULT_VIEWPORT: dict[str,
                        int] = {
                            "width": 1920,

cmdlet/file/search_engines.py

@@ -0,0 +1,994 @@
+"""Search engine result parsing — Bing, DuckDuckGo, Yahoo, and site crawling.
+
+Extracted from search.py to keep the cmdlet class focused on orchestration.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence
+from collections import deque
+from pathlib import Path
+import re
+import time
+import html as _html
+from urllib.parse import urlparse, parse_qs, unquote, urljoin
+
+from SYS.logger import debug
+from SYS.payload_builders import normalize_file_extension
+
+_WHITESPACE_RE = re.compile(r"\s+")
+_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE)
+_FILETYPE_TOKEN_RE = re.compile(
+    r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b",
+    flags=re.IGNORECASE,
+)
+_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE)
+_FILETYPE_REMOVE_RE = re.compile(
+    r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b",
+    flags=re.IGNORECASE,
+)
+_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:")
+_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE)
+_HTML_TAG_RE = re.compile(r"<[^>]+>")
+_DDG_RESULT_ANCHOR_RE = re.compile(
+    r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
+    flags=re.IGNORECASE | re.DOTALL,
+)
+_GENERIC_ANCHOR_RE = re.compile(
+    r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>',
+    flags=re.IGNORECASE | re.DOTALL,
+)
+_BING_RESULT_ANCHOR_RE = re.compile(
+    r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
+    flags=re.IGNORECASE | re.DOTALL,
+)
+
+
+def _normalize_extension(ext_value: Any) -> str:
+    """Sanitize extension strings to alphanumerics and cap at 5 chars."""
+    return normalize_file_extension(ext_value)
+
+
+def _normalize_host(value: Any) -> str:
+    """Normalize host names for matching/filtering."""
+    host = str(value or "").strip().lower()
+    if host.startswith("www."):
+        host = host[4:]
+    if ":" in host:
+        host = host.split(":", 1)[0]
+    return host
+
+
+def _normalize_space(text: Any) -> str:
+    return _WHITESPACE_RE.sub(" ", str(text or "")).strip()
+
+
+def _url_matches_site(cls, url: str, site_host: str) -> bool:
+    """Return True when URL host is the requested site/subdomain."""
+    try:
+        parsed = urlparse(str(url or ""))
+        host = _normalize_host(getattr(parsed, "hostname", "") or "")
+    except Exception:
+        return False
+
+    target = _normalize_host(site_host)
+    if not host or not target:
+        return False
+    return host == target or host.endswith(f".{target}")
+
+
+def _itertext_join(node: Any) -> str:
+    try:
+        return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()])
+    except Exception:
+        return ""
+
+
+def _html_fragment_to_text(fragment: Any) -> str:
+    text = _HTML_TAG_RE.sub(" ", str(fragment or ""))
+    return _html.unescape(text)
+
+
+def _append_web_result(
+    cls,
+    items: List[Dict[str, str]],
+    seen_urls: set[str],
+    *,
+    site_host: str,
+    url_text: str,
+    title_text: str,
+    snippet_text: str,
+) -> None:
+    url_clean = str(url_text or "").strip()
+    if not url_clean or not url_clean.startswith(("http://", "https://")):
+        return
+    if not _url_matches_site(url_clean, site_host):
+        return
+    if url_clean in seen_urls:
+        return
+
+    seen_urls.add(url_clean)
+    items.append(
+        {
+            "url": url_clean,
+            "title": _normalize_space(title_text) or url_clean,
+            "snippet": _normalize_space(snippet_text),
+        }
+    )
+
+
+def _parse_web_results_with_fallback(
+    cls,
+    *,
+    html_text: str,
+    limit: int,
+    lxml_parser: Any,
+    regex_parser: Any,
+    fallback_when_empty: bool = False,
+) -> List[Dict[str, str]]:
+    """Run an lxml-based parser with an optional regex fallback."""
+    items: List[Dict[str, str]] = []
+    seen_urls: set[str] = set()
+    should_run_regex = False
+
+    try:
+        from lxml import html as lxml_html
+
+        doc = lxml_html.fromstring(html_text or "")
+        lxml_parser(doc, items, seen_urls)
+        should_run_regex = fallback_when_empty and not items
+    except Exception:
+        should_run_regex = True
+
+    if should_run_regex:
+        regex_parser(html_text or "", items, seen_urls)
+
+    return items[:limit]
+
+
+def _extract_duckduckgo_target_url(href: Any) -> str:
+    """Extract direct target URL from DuckDuckGo result links."""
+    raw_href = str(href or "").strip()
+    if not raw_href:
+        return ""
+
+    if raw_href.startswith("//"):
+        raw_href = f"https:{raw_href}"
+
+    if raw_href.startswith("/"):
+        raw_href = f"https://duckduckgo.com{raw_href}"
+
+    parsed = None
+    try:
+        parsed = urlparse(raw_href)
+    except Exception:
+        parsed = None
+
+    try:
+        host = str(getattr(parsed, "hostname", "") or "").strip().lower()
+    except Exception:
+        host = ""
+
+    if host.endswith("duckduckgo.com"):
+        try:
+            query = parse_qs(str(getattr(parsed, "query", "") or ""))
+            candidate = (query.get("uddg") or [""])[0]
+            if candidate:
+                return str(unquote(candidate)).strip()
+        except Exception:
+            pass
+
+    return raw_href
+
+
+def _extract_yahoo_target_url(href: Any) -> str:
+    """Extract direct target URL from Yahoo redirect links."""
+    raw_href = str(href or "").strip()
+    if not raw_href:
+        return ""
+
+    ru_match = _YAHOO_RU_RE.search(raw_href)
+    if ru_match:
+        try:
+            return str(unquote(ru_match.group(1))).strip()
+        except Exception:
+            pass
+
+    try:
+        parsed = urlparse(raw_href)
+        query = parse_qs(str(getattr(parsed, "query", "") or ""))
+        candidate = (query.get("RU") or query.get("ru") or [""])[0]
+        if candidate:
+            return str(unquote(candidate)).strip()
+    except Exception:
+        pass
+
+    return raw_href
+
+
+def parse_duckduckgo_results(
+    cls,
+    *,
+    html_text: str,
+    site_host: str,
+    limit: int,
+) -> List[Dict[str, str]]:
+    """Parse DuckDuckGo HTML results into normalized rows."""
+    def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        result_nodes = doc.xpath("//div[contains(@class, 'result')]")
+
+        for node in result_nodes:
+            links = node.xpath(".//a[contains(@class, 'result__a')]")
+            if not links:
+                continue
+
+            link = links[0]
+            href = _extract_duckduckgo_target_url(link.get("href"))
+            title = _itertext_join(link)
+
+            snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]")
+            snippet = ""
+            if snippet_nodes:
+                snippet = _itertext_join(snippet_nodes[0])
+
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text=snippet,
+            )
+            if len(items) >= limit:
+                break
+
+    def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html):
+            href = _extract_duckduckgo_target_url(match.group(1))
+            title_html = match.group(2)
+            title = _html_fragment_to_text(title_html)
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text="",
+            )
+            if len(items) >= limit:
+                break
+
+    return _parse_web_results_with_fallback(
+        cls,
+        html_text=html_text,
+        limit=limit,
+        lxml_parser=_parse_lxml,
+        regex_parser=_parse_regex,
+        fallback_when_empty=True,
+    )
+
+
+def parse_yahoo_results(
+    cls,
+    *,
+    html_text: str,
+    site_host: str,
+    limit: int,
+) -> List[Dict[str, str]]:
+    """Parse Yahoo HTML search results into normalized rows."""
+    def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        for node in doc.xpath("//a[@href]"):
+            href = _extract_yahoo_target_url(node.get("href"))
+            title = _itertext_join(node)
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text="",
+            )
+            if len(items) >= limit:
+                break
+
+    def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        for match in _GENERIC_ANCHOR_RE.finditer(raw_html):
+            href = _extract_yahoo_target_url(match.group(1))
+            title_html = match.group(2)
+            title = _html_fragment_to_text(title_html)
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text="",
+            )
+            if len(items) >= limit:
+                break
+
+    return _parse_web_results_with_fallback(
+        cls,
+        html_text=html_text,
+        limit=limit,
+        lxml_parser=_parse_lxml,
+        regex_parser=_parse_regex,
+    )
+
+
+def query_yahoo(
+    cls,
+    *,
+    search_query: str,
+    site_host: str,
+    limit: int,
+    session: Any,
+    deadline: Optional[float] = None,
+) -> List[Dict[str, str]]:
+    """Fetch results from Yahoo search (robust fallback in bot-protected envs)."""
+    all_rows: List[Dict[str, str]] = []
+    seen_urls: set[str] = set()
+
+    max_pages = max(1, min((max(1, int(limit or 1)) + 9) // 10, 3))
+    for page_idx in range(max_pages):
+        if deadline is not None and time.monotonic() >= deadline:
+            break
+
+        params = {
+            "p": search_query,
+            "n": "10",
+            "b": str((page_idx * 10) + 1),
+        }
+        try:
+            read_timeout = 10.0
+            if deadline is not None:
+                remaining = max(0.0, float(deadline - time.monotonic()))
+                if remaining <= 0.0:
+                    break
+                read_timeout = max(3.0, min(10.0, remaining))
+
+            response = session.get(
+                "https://search.yahoo.com/search",
+                params=params,
+                timeout=(3, read_timeout),
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/124.0.0.0 Safari/537.36"
+                    ),
+                    "Accept-Language": "en-US,en;q=0.9",
+                },
+            )
+            response.raise_for_status()
+        except Exception:
+            break
+
+        page_rows = parse_yahoo_results(
+            cls,
+            html_text=response.text,
+            site_host=site_host,
+            limit=max(1, limit - len(all_rows)),
+        )
+        new_rows = 0
+        for row in page_rows:
+            url_value = str(row.get("url") or "").strip()
+            if not url_value or url_value in seen_urls:
+                continue
+            seen_urls.add(url_value)
+            all_rows.append(row)
+            new_rows += 1
+            if len(all_rows) >= limit:
+                break
+
+        if len(all_rows) >= limit or new_rows == 0:
+            break
+
+    return all_rows[:limit]
+
+
+def parse_bing_results(
+    cls,
+    *,
+    html_text: str,
+    site_host: str,
+    limit: int,
+) -> List[Dict[str, str]]:
+    """Parse Bing HTML search results into normalized rows."""
+    def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]")
+
+        for node in result_nodes:
+            links = node.xpath(".//h2/a")
+            if not links:
+                continue
+            link = links[0]
+            href = str(link.get("href") or "").strip()
+            title = _itertext_join(link)
+
+            snippet = ""
+            for sel in (
+                ".//*[contains(@class,'b_caption')]//p",
+                ".//*[contains(@class,'b_snippet')]",
+                ".//p",
+            ):
+                snip_nodes = node.xpath(sel)
+                if snip_nodes:
+                    snippet = _itertext_join(snip_nodes[0])
+                    break
+
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text=snippet,
+            )
+            if len(items) >= limit:
+                break
+
+    def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
+        for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html):
+            href = match.group(1)
+            title = _html_fragment_to_text(match.group(2))
+            _append_web_result(
+                cls,
+                items,
+                seen_urls,
+                site_host=site_host,
+                url_text=href,
+                title_text=title,
+                snippet_text="",
+            )
+            if len(items) >= limit:
+                break
+
+    return _parse_web_results_with_fallback(
+        cls,
+        html_text=html_text,
+        limit=limit,
+        lxml_parser=_parse_lxml,
+        regex_parser=_parse_regex,
+    )
+
+
+def query_bing(
+    cls,
+    *,
+    search_query: str,
+    site_host: str,
+    limit: int,
+    session: Any,
+    deadline: Optional[float] = None,
+) -> List[Dict[str, str]]:
+    """Fetch results from Bing (supports filetype: and site: natively)."""
+    all_rows: List[Dict[str, str]] = []
+    seen_urls: set[str] = set()
+
+    page_start = 1
+    pages_checked = 0
+    max_pages = max(1, min((max(1, int(limit or 1)) + 49) // 50, 3))
+    while len(all_rows) < limit and pages_checked < max_pages:
+        if deadline is not None and time.monotonic() >= deadline:
+            break
+
+        params = {"q": search_query, "first": str(page_start), "count": "50"}
+        try:
+            read_timeout = 10.0
+            if deadline is not None:
+                remaining = max(0.0, float(deadline - time.monotonic()))
+                if remaining <= 0.0:
+                    break
+                read_timeout = max(3.0, min(10.0, remaining))
+
+            response = session.get(
+                "https://www.bing.com/search",
+                params=params,
+                timeout=(3, read_timeout),
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/124.0.0.0 Safari/537.36"
+                    ),
+                    "Accept-Language": "en-US,en;q=0.9",
+                },
+            )
+            response.raise_for_status()
+        except Exception:
+            break
+
+        page_rows = parse_bing_results(
+            cls,
+            html_text=response.text,
+            site_host=site_host,
+            limit=max(1, limit - len(all_rows)),
+        )
+        new_rows = 0
+        for row in page_rows:
+            url_value = str(row.get("url") or "").strip()
+            if not url_value or url_value in seen_urls:
+                continue
+            seen_urls.add(url_value)
+            all_rows.append(row)
+            new_rows += 1
+            if len(all_rows) >= limit:
+                break
+
+        if new_rows == 0 or len(all_rows) >= limit:
+            break
+        page_start += 50
+        pages_checked += 1
+
+    return all_rows
+
+
+def query_web_search(
+    cls,
+    *,
+    search_query: str,
+    site_host: str,
+    limit: int,
+) -> List[Dict[str, str]]:
+    """Execute web search and return parsed result rows.
+
+    Uses Yahoo first (works in environments where Bing/DDG HTML endpoints
+    are challenge-gated), then Bing, then DuckDuckGo.
+    """
+    from API.requests_client import get_requests_session
+
+    session = get_requests_session()
+    normalized_limit = max(1, min(int(limit or 1), 100))
+    engine_deadline = time.monotonic() + 12.0
+
+    all_rows = query_yahoo(
+        cls,
+        search_query=search_query,
+        site_host=site_host,
+        limit=normalized_limit,
+        session=session,
+        deadline=engine_deadline,
+    )
+    if all_rows:
+        return all_rows[:normalized_limit]
+
+    all_rows = query_bing(
+        cls,
+        search_query=search_query,
+        site_host=site_host,
+        limit=normalized_limit,
+        session=session,
+        deadline=engine_deadline,
+    )
+    if all_rows:
+        return all_rows[:normalized_limit]
+
+    all_rows_ddg: List[Dict[str, str]] = []
+    seen_urls: set[str] = set()
+    endpoints = [
+        "https://html.duckduckgo.com/html/",
+        "https://duckduckgo.com/html/",
+    ]
+    for endpoint in endpoints:
+        if time.monotonic() >= engine_deadline:
+            break
+        max_offsets = min(3, max(1, (normalized_limit + 29) // 30))
+        for page_idx in range(max_offsets):
+            if time.monotonic() >= engine_deadline:
+                break
+            offset = page_idx * 30
+            params = {"q": search_query, "s": str(offset)}
+            remaining = max(0.0, float(engine_deadline - time.monotonic()))
+            if remaining <= 0.0:
+                break
+            read_timeout = max(3.0, min(10.0, remaining))
+            response = session.get(
+                endpoint,
+                params=params,
+                timeout=(3, read_timeout),
+                headers={"Referer": "https://duckduckgo.com/"},
+            )
+            response.raise_for_status()
+            page_rows = parse_duckduckgo_results(
+                cls,
+                html_text=response.text,
+                site_host=site_host,
+                limit=max(1, normalized_limit - len(all_rows_ddg)),
+            )
+            new_rows = 0
+            for row in page_rows:
+                url_value = str(row.get("url") or "").strip()
+                if not url_value or url_value in seen_urls:
+                    continue
+                seen_urls.add(url_value)
+                all_rows_ddg.append(row)
+                new_rows += 1
+                if len(all_rows_ddg) >= normalized_limit:
+                    break
+            if len(all_rows_ddg) >= normalized_limit or new_rows == 0:
+                break
+        if all_rows_ddg:
+            break
+
+    return all_rows_ddg[:normalized_limit]
+
+
+def _is_probable_html_path(path_value: str) -> bool:
+    """Return True when URL path likely points to an HTML page."""
+    path = str(path_value or "").strip()
+    if not path:
+        return True
+    suffix = Path(path).suffix.lower()
+    if not suffix:
+        return True
+    return suffix in {".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".shtml", ".xhtml"}
+
+
+def _extract_html_links(cls, *, html_text: str, base_url: str) -> List[str]:
+    """Extract absolute links from an HTML document."""
+    links: List[str] = []
+    seen: set[str] = set()
+
+    def _add_link(raw_href: Any) -> None:
+        href = str(raw_href or "").strip()
+        if not href or href.startswith(("#", "javascript:", "mailto:")):
+            return
+        try:
+            absolute = urljoin(base_url, href)
+            parsed = urlparse(absolute)
+        except Exception:
+            return
+        if str(getattr(parsed, "scheme", "") or "").lower() not in {"http", "https"}:
+            return
+        clean = parsed._replace(fragment="").geturl()
+        if clean in seen:
+            return
+        seen.add(clean)
+        links.append(clean)
+
+    try:
+        from lxml import html as lxml_html
+
+        doc = lxml_html.fromstring(html_text or "")
+        for node in doc.xpath("//a[@href]"):
+            _add_link(node.get("href"))
+    except Exception:
+        href_pattern = re.compile(r'<a[^>]+href=["\']([^"\']+)["\']', flags=re.IGNORECASE)
+        for match in href_pattern.finditer(html_text or ""):
+            _add_link(match.group(1))
+
+    return links
+
+
+def crawl_site_for_extension(
+    cls,
+    *,
+    seed_url: str,
+    site_host: str,
+    extension: str,
+    limit: int,
+    max_duration_seconds: float = 15.0,
+) -> List[Dict[str, str]]:
+    """Fallback crawler that discovers in-site file links by extension."""
+    from API.requests_client import get_requests_session
+
+    normalized_ext = _normalize_extension(extension)
+    if not normalized_ext:
+        return []
+
+    start_url = _normalize_seed_url(cls, seed_url, site_host)
+    if not start_url:
+        return []
+
+    session = get_requests_session()
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/124.0.0.0 Safari/537.36"
+        ),
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    queue: deque[str] = deque([start_url])
+    queued: set[str] = {start_url}
+    visited_pages: set[str] = set()
+    seen_files: set[str] = set()
+    rows: List[Dict[str, str]] = []
+    normalized_limit = max(1, min(int(limit or 1), 100))
+    max_pages = max(8, min(normalized_limit * 4, 64))
+    crawl_deadline = time.monotonic() + max(5.0, float(max_duration_seconds or 0.0))
+
+    while (
+        queue
+        and len(visited_pages) < max_pages
+        and len(rows) < normalized_limit
+        and time.monotonic() < crawl_deadline
+    ):
+        page_url = queue.popleft()
+        queued.discard(page_url)
+        if page_url in visited_pages:
+            continue
+        visited_pages.add(page_url)
+
+        if time.monotonic() >= crawl_deadline:
+            break
+
+        try:
+            response = session.get(page_url, timeout=(4, 8), headers=headers)
+            response.raise_for_status()
+        except Exception:
+            continue
+
+        final_url = str(getattr(response, "url", "") or page_url)
+        try:
+            parsed_final = urlparse(final_url)
+        except Exception:
+            continue
+
+        final_host = _normalize_host(getattr(parsed_final, "hostname", "") or "")
+        if not _url_matches_site(final_url, site_host):
+            continue
+
+        final_path = str(getattr(parsed_final, "path", "") or "")
+        direct_ext = _normalize_extension(Path(final_path).suffix)
+        if direct_ext == normalized_ext:
+            file_url = parsed_final._replace(fragment="").geturl()
+            if file_url not in seen_files:
+                seen_files.add(file_url)
+                title = Path(unquote(final_path)).name or file_url
+                rows.append(
+                    {
+                        "url": file_url,
+                        "title": title,
+                        "snippet": "Discovered via in-site crawl",
+                    }
+                )
+            continue
+
+        content_type = str((response.headers or {}).get("content-type", "") or "").lower()
+        if "html" not in content_type and "xhtml" not in content_type:
+            continue
+
+        html_text = str(getattr(response, "text", "") or "")
+        if not html_text:
+            continue
+        if len(html_text) > 2_500_000:
+            continue
+
+        discovered_links = _extract_html_links(cls, html_text=html_text, base_url=final_url)
+        for idx, target in enumerate(discovered_links):
+            if len(rows) >= normalized_limit:
+                break
+            if idx >= 300:
+                break
+            if time.monotonic() >= crawl_deadline:
+                break
+            try:
+                parsed_target = urlparse(target)
+            except Exception:
+                continue
+            target_host = _normalize_host(getattr(parsed_target, "hostname", "") or "")
+            if not target_host or not (target_host == final_host or target_host.endswith(f".{site_host}")):
+                if not _url_matches_site(target, site_host):
+                    continue
+
+            target_clean = parsed_target._replace(fragment="").geturl()
+            target_path = str(getattr(parsed_target, "path", "") or "")
+            target_ext = _normalize_extension(Path(target_path).suffix)
+
+            if target_ext == normalized_ext:
+                if target_clean in seen_files:
+                    continue
+                seen_files.add(target_clean)
+                title = Path(unquote(target_path)).name or target_clean
+                rows.append(
+                    {
+                        "url": target_clean,
+                        "title": title,
+                        "snippet": f"Discovered via crawl from {final_path or '/'}",
+                    }
+                )
+                continue
+
+            if _is_probable_html_path(target_path):
+                if target_clean not in visited_pages and target_clean not in queued:
+                    queue.append(target_clean)
+                    queued.add(target_clean)
+
+    if time.monotonic() >= crawl_deadline:
+        debug(
+            "Web crawl fallback reached time budget",
+            {
+                "site": site_host,
+                "visited_pages": len(visited_pages),
+                "queued_pages": len(queue),
+                "results": len(rows),
+                "time_budget_seconds": max_duration_seconds,
+            },
+        )
+
+    return rows[:normalized_limit]
+
+
+def _extract_site_host(cls, candidate: Any) -> Optional[str]:
+    """Extract a host/domain from URL-like input."""
+    raw = str(candidate or "").strip().strip('"').strip("'")
+    if not raw:
+        return None
+
+    if raw.lower().startswith("site:"):
+        raw = raw.split(":", 1)[1].strip()
+
+    parsed = None
+    try:
+        parsed = urlparse(raw)
+    except Exception:
+        parsed = None
+
+    if parsed is None or not getattr(parsed, "hostname", None):
+        try:
+            parsed = urlparse(f"https://{raw}")
+        except Exception:
+            parsed = None
+
+    host = ""
+    try:
+        host = str(getattr(parsed, "hostname", "") or "").strip().lower()
+    except Exception:
+        host = ""
+
+    host = _normalize_host(host)
+    if not host or "." not in host:
+        return None
+    return host
+
+
+def _normalize_seed_url(cls, seed_value: Any, site_host: str) -> str:
+    """Build a safe crawl starting URL from user input and resolved host."""
+    raw = str(seed_value or "").strip().strip("'\"")
+    if not raw:
+        raw = str(site_host or "").strip()
+
+    if raw and not raw.startswith(("http://", "https://")):
+        raw = f"https://{raw}"
+
+    try:
+        parsed = urlparse(raw)
+    except Exception:
+        parsed = urlparse("")
+
+    target = _normalize_host(site_host)
+    host = _normalize_host(getattr(parsed, "hostname", "") or "")
+    if target and host and not (host == target or host.endswith(f".{target}")):
+        return f"https://{target}/"
+
+    scheme = str(getattr(parsed, "scheme", "") or "https").lower()
+    if scheme not in {"http", "https"}:
+        scheme = "https"
+
+    netloc = str(getattr(parsed, "netloc", "") or "").strip()
+    if not netloc:
+        netloc = target
+    path = str(getattr(parsed, "path", "") or "").strip()
+    if not path:
+        path = "/"
+
+    return f"{scheme}://{netloc}{path}"
+
+
+def build_web_search_plan(
+    cls,
+    *,
+    query: str,
+    positional_args: List[str],
+    storage_backend: Optional[str],
+    store_filter: Optional[str],
+    hash_query: List[str],
+) -> Optional[Dict[str, Any]]:
+    """Build web-search plan for URL + ext/filetype query syntax.
+
+    Example input:
+        search-file "example.com/foo" -query "ext:pdf"
+    Produces:
+        site:example.com filetype:pdf
+    """
+    if storage_backend or store_filter or hash_query:
+        return None
+
+    text = _normalize_space(query)
+    if not text:
+        return None
+
+    local_markers = ("url:", "hash:", "tag:", "instance:", "system:")
+    if any(marker in text.lower() for marker in local_markers):
+        return None
+
+    site_host: Optional[str] = None
+    site_from_positional = False
+    site_token_to_strip = ""
+    seed_url = ""
+
+    site_match = _SITE_TOKEN_RE.search(text)
+    if site_match:
+        site_host = _extract_site_host(cls, site_match.group(1))
+        seed_url = str(site_match.group(1) or "").strip()
+
+    if not site_host and positional_args:
+        site_host = _extract_site_host(cls, positional_args[0])
+        site_from_positional = bool(site_host)
+        if site_from_positional:
+            site_token_to_strip = str(positional_args[0] or "").strip()
+            seed_url = site_token_to_strip
+
+    if not site_host:
+        for token in text.split():
+            candidate = str(token or "").strip().strip(",")
+            if not candidate:
+                continue
+            lower_candidate = candidate.lower()
+            if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")):
+                continue
+            if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith(
+                ("http://", "https://")
+            ):
+                continue
+            guessed = _extract_site_host(cls, candidate)
+            if guessed:
+                site_host = guessed
+                site_token_to_strip = candidate
+                break
+
+    if not site_host:
+        return None
+
+    filetype_match = _FILETYPE_TOKEN_RE.search(text)
+    filetype = _normalize_extension(filetype_match.group(1)) if filetype_match else ""
+
+    has_explicit_site = bool(site_match)
+    if not filetype and not has_explicit_site:
+        return None
+
+    residual = text
+    residual = _SITE_REMOVE_RE.sub(" ", residual)
+    residual = _FILETYPE_REMOVE_RE.sub(" ", residual)
+
+    if site_from_positional and positional_args:
+        first = str(positional_args[0] or "").strip()
+        if first:
+            residual = re.sub(rf"(?:^|\s){re.escape(first)}(?:\s|$)", " ", residual, count=1)
+    elif site_token_to_strip:
+        residual = re.sub(
+            rf"(?:^|\s){re.escape(site_token_to_strip)}(?:\s|$)",
+            " ",
+            residual,
+            count=1,
+        )
+
+    residual = _normalize_space(residual)
+
+    search_terms: List[str] = [f"site:{site_host}"]
+    if filetype:
+        search_terms.append(f"filetype:{filetype}")
+    if residual:
+        search_terms.append(residual)
+
+    search_query = " ".join(search_terms).strip()
+    if not search_query:
+        return None
+
+    normalized_seed_url = _normalize_seed_url(cls, seed_url, site_host)
+
+    return {
+        "site_host": site_host,
+        "filetype": filetype,
+        "search_query": search_query,
+        "residual": residual,
+        "seed_url": normalized_seed_url,
+    }

cmdlet/get_url.py

@@ -475,7 +475,7 @@ class Get_Url(Cmdlet):
 
             publish_result_table(ctx, table if display_items else None, display_items, subject=result)
             
-            # Emit after table state is finalized to prevent side effects in TUI rendering
+            # Emit after table state is finalized
             for d in display_items:
                 ctx.emit(d)
 

cmdlet/metadata/relationship_add.py

@@ -60,8 +60,7 @@ CMDLET = Cmdlet(
         "- Mode 2: Use -king to explicitly set which item/hash is the king: @1-3 | add-relationship -king @4",
         "- Mode 2b: Use -king and -alt to select both sides from the last table: add-relationship -king @1 -alt @3-5",
         "- Mode 3: Read relationships from sidecar tags:",
-        "    - New format: 'relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>' (first hash is king)",
+        "    - Format: 'relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>' (first hash is king)",
-        "    - Legacy:     'relationship: hash(king)<HASH>,hash(alt)<HASH>...'",
         "- Supports three relationship types: king (primary), alt (alternative), related (other versions)",
         "- When using -king, all piped items become the specified relationship type to the king",
     ],
@@ -71,13 +70,10 @@ CMDLET = Cmdlet(
 _normalize_hash_hex = sh.normalize_hash
 
 
-def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]:
+    def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]:
     """Parse relationship tags.
 
-    Supported formats:
+    Format: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>
-    - New:  relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>
-    - Old:  relationship: hash(king)<HASH>,hash(alt)<HASH>...
-
     Returns a dict like {"king": ["HASH1"], "alt": ["HASH2"], ...}
     """
     result: Dict[str,
@@ -85,20 +81,7 @@ def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]:
     if not isinstance(tag_value, str):
         return result
 
-    # Match patterns like hash(king)HASH or hash(type)<HASH>
+    # Extract hashes; first is king
-    pattern = r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?"
-    matches = re.findall(pattern, tag_value)
-
-    if matches:
-        for rel_type, hash_value in matches:
-            normalized = _normalize_hash_hex(hash_value)
-            if normalized:
-                if rel_type not in result:
-                    result[rel_type] = []
-                result[rel_type].append(normalized)
-        return result
-
-    # New format: extract hashes, first is king
     hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", tag_value)
     hashes = [h.strip().lower() for h in hashes if isinstance(h, str)]
     if not hashes:
@@ -833,7 +816,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
             hydrus_client.set_relationship(h, king_hash, str(rel_type))
         return 0
 
-    # Process each item in the list (legacy path-based mode)
+    # Process each item in the list (path-based mode)
     for item in items_to_process:
         # Extract hash and path from current item
         file_hash = None
@@ -1033,136 +1016,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
 
     return 0
 
-    # FILE MODE: Read relationships from sidecar (legacy mode - for -path arg only)
-    log(
-        "Note: Use piping mode for easier relationships. Example: 1,2,3 | add-relationship",
-        file=sys.stderr,
-    )
-
-    # Resolve media path from -path arg or result target
-    target = getattr(result, "target", None) or getattr(result, "path", None)
-    media_path = (
-        arg_path
-        if arg_path is not None else Path(str(target)) if isinstance(target,
-                                                                     str) else None
-    )
-    if media_path is None:
-        log("Provide -path <file> or pipe a local file result", file=sys.stderr)
-        return 1
-
-    # Validate local file
-    if str(media_path).lower().startswith(("http://", "https://")):
-        log("This cmdlet requires a local file path, not a URL", file=sys.stderr)
-        return 1
-    if not media_path.exists() or not media_path.is_file():
-        log(f"File not found: {media_path}", file=sys.stderr)
-        return 1
-
-    # Build Hydrus client
-    hydrus_provider = get_plugin("hydrusnetwork", config)
-    try:
-        hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None
-    except Exception as exc:
-        log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
-        return 1
-
-    if hydrus_client is None:
-        log("Hydrus client unavailable", file=sys.stderr)
-        return 1
-
-    # Read sidecar to find relationship tags
-    sidecar_path = find_sidecar(media_path)
-    if sidecar_path is None:
-        log(f"No sidecar found for {media_path.name}", file=sys.stderr)
-        return 1
-
-    try:
-        _, tags, _ = read_sidecar(sidecar_path)
-    except Exception as exc:
-        log(f"Failed to read sidecar: {exc}", file=sys.stderr)
-        return 1
-
-    # Find relationship tags (format: "relationship: hash(king)<HASH>,hash(alt)<HASH>,hash(related)<HASH>")
-    relationship_tags = [
-        t for t in tags if isinstance(t, str) and t.lower().startswith("relationship:")
-    ]
-
-    if not relationship_tags:
-        log("No relationship tags found in sidecar", file=sys.stderr)
-        return 0  # Not an error, just nothing to do
-
-    # Get the file hash from result (should have been set by add-file)
-    file_hash = getattr(result, "hash_hex", None)
-    if not file_hash:
-        log("File hash not available (run add-file first)", file=sys.stderr)
-        return 1
-
-    file_hash = _normalize_hash_hex(file_hash)
-    if not file_hash:
-        log("Invalid file hash format", file=sys.stderr)
-        return 1
-
-    # Parse relationships from tags and apply them
-    success_count = 0
-    error_count = 0
-
-    for rel_tag in relationship_tags:
-        try:
-            # Parse: "relationship: hash(king)<HASH>,hash(alt)<HASH>,hash(related)<HASH>"
-            rel_str = rel_tag.split(":", 1)[1].strip()  # Get part after "relationship:"
-
-            # Parse relationships
-            rels = _extract_relationships_from_tag(f"relationship: {rel_str}")
-
-            # Set the relationships in Hydrus
-            for rel_type, related_hashes in rels.items():
-                if not related_hashes:
-                    continue
-
-                for related_hash in related_hashes:
-                    # Don't set relationship between hash and itself
-                    if file_hash == related_hash:
-                        continue
-
-                    try:
-                        hydrus_client.set_relationship(
-                            file_hash,
-                            related_hash,
-                            rel_type
-                        )
-                        log(
-                            f"[add-relationship] Set {rel_type} relationship: "
-                            f"{file_hash} <-> {related_hash}",
-                            file=sys.stderr,
-                        )
-                        success_count += 1
-                    except Exception as exc:
-                        log(
-                            f"Failed to set {rel_type} relationship: {exc}",
-                            file=sys.stderr
-                        )
-                        error_count += 1
-
-        except Exception as exc:
-            log(f"Failed to parse relationship tag: {exc}", file=sys.stderr)
-            error_count += 1
-
-    if success_count > 0:
-        log(
-            f"Successfully set {success_count} relationship(s) for {media_path.name}",
-            file=sys.stderr,
-        )
-        ctx.emit(
-            f"add-relationship: {media_path.name} ({success_count} relationships set)"
-        )
-        return 0
-    elif error_count == 0:
-        log("No relationships to set", file=sys.stderr)
-        return 0  # Success with nothing to do
-    else:
-        log(f"Failed with {error_count} error(s)", file=sys.stderr)
-        return 1
-
 
 # Register cmdlet (no legacy decorator)
 CMDLET.exec = _run

cmdlet/metadata/tag_add.py

@@ -1081,6 +1081,8 @@ class Add_Tag(Cmdlet):
                     ok_add = True
 
             if not queued_bulk:
+                ok_add = False
+                ok_remove = True
                 try:
                     ok_add = backend.add_tag(
                         resolved_hash,
@@ -1088,11 +1090,19 @@ class Add_Tag(Cmdlet):
                         config=config,
                         existing_tags=existing_tag_list,
                     )
-                    if not ok_add:
-                        log("[add_tag] Warning: Store rejected tag update", file=sys.stderr)
                 except Exception as exc:
                     log(f"[add_tag] Warning: Failed adding tag: {exc}", file=sys.stderr)
                     ok_add = False
+                if tags_to_remove:
+                    try:
+                        delete_fn = getattr(backend, "delete_tag", None)
+                        if callable(delete_fn):
+                            ok_remove = delete_fn(resolved_hash, list(tags_to_remove), config=config)
+                    except Exception as exc:
+                        log(f"[add_tag] Warning: Failed removing tag: {exc}", file=sys.stderr)
+                        ok_remove = False
+                if not ok_add and not ok_remove:
+                    log("[add_tag] Warning: Store rejected tag update", file=sys.stderr)
 
             if ok_add and merged_tags:
                 refreshed_list = list(merged_tags)

cmdnat/config.py

@@ -1158,8 +1158,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
     if not args:
         if sys.stdin.isatty() and not piped_result:
             print(
-                "Interactive TUI config editor has been discontinued. "
+                "Run `.config <key>` to view or edit a specific setting. "
-                "Showing configuration table instead."
             )
         return _show_config_table(current_config)
 

cmdnat/status.py

@@ -60,8 +60,7 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int:
         _add_startup_check(startup_table, "ERROR", "STATUS", detail=str(exc))
 
     if startup_table.rows:
-        # Mark as rendered to prevent CLI.py from auto-printing it to stdout 
+        # Mark as rendered to prevent CLI.py from auto-printing it to stdout
-        # (avoiding duplication in TUI logs, while keeping it in TUI Results)
         setattr(startup_table, "_rendered_by_cmdlet", True)
         ctx.set_current_stage_table(startup_table)
     

plugins/local/__init__.py

@@ -4,11 +4,27 @@ import shutil
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
-from PluginCore.base import Provider
+from PluginCore.base import Provider, SearchResult
-from SYS.metadata import write_metadata, write_tags
+from SYS.metadata import _read_sidecar_metadata, read_tags_from_file, write_metadata, write_tags
 from SYS.utils import sanitize_filename, sha256_file, unique_path
 
 
+def _format_size_safe(size_bytes: Any) -> str:
+    if size_bytes is None:
+        return ""
+    try:
+        size = int(size_bytes)
+    except (TypeError, ValueError):
+        return str(size_bytes or "")
+    if size < 1024:
+        return f"{size} B"
+    if size < 1024 * 1024:
+        return f"{size / 1024:.1f} KB"
+    if size < 1024 * 1024 * 1024:
+        return f"{size / (1024 * 1024):.1f} MB"
+    return f"{size / (1024 * 1024 * 1024):.2f} GB"
+
+
 def _copy_sidecars(source_path: Path, target_path: Path) -> None:
     possible_sidecars = [
         source_path.with_suffix(source_path.suffix + ".json"),
@@ -80,7 +96,7 @@ class Local(Provider):
     PLUGIN_NAME = "local"
     PLUGIN_ALIASES = ("filesystem", "fs")
     MULTI_INSTANCE = True
-    SUPPORTED_CMDLETS = frozenset({"add-file"})
+    SUPPORTED_CMDLETS = frozenset({"add-file", "search-file"})
 
     @property
     def label(self) -> str:
@@ -165,6 +181,152 @@ class Local(Provider):
     def validate(self) -> bool:
         return True
 
+    @staticmethod
+    def _infer_media_kind(ext: str) -> str:
+        e = str(ext or "").strip().lower().lstrip(".")
+        if e in {"mp3", "flac", "wav", "ogg", "opus", "m4a", "aac", "wma", "aiff"}:
+            return "audio"
+        if e in {"mp4", "mkv", "avi", "mov", "webm", "wmv", "flv", "m4v"}:
+            return "video"
+        if e in {"pdf", "epub", "mobi", "azw3", "djvu", "cbr", "cbz", "chm"}:
+            return "book"
+        if e in {"zip", "rar", "7z", "tar", "gz", "bz2", "xz", "iso"}:
+            return "archive"
+        if e in {"exe", "msi", "apk", "deb", "rpm", "appimage", "bin"}:
+            return "software"
+        if e in {"jpg", "jpeg", "png", "gif", "bmp", "webp", "svg", "tiff", "ico"}:
+            return "image"
+        return "file"
+
+    def search(
+        self,
+        query: str,
+        limit: int = 50,
+        filters: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> List[SearchResult]:
+        q = str(query or "").strip()
+        match_all = not q or q == "*"
+        query_tokens = [t.lower() for t in q.split()] if not match_all else []
+        max_results = max(1, int(limit))
+
+        target_instance = None
+        if filters:
+            target_instance = str(filters.get("instance") or filters.get("store") or "").strip() or None
+
+        results: List[SearchResult] = []
+        instances = self.plugin_instance_configs()
+
+        for inst_name, inst_cfg in instances.items():
+            if target_instance and str(inst_name or "").strip().lower() != target_instance.lower():
+                continue
+
+            settings = self._settings_from_config(inst_cfg, instance_name=inst_name)
+            root_text = str(settings.get("path") or "").strip()
+            if not root_text:
+                continue
+
+            root = Path(root_text).expanduser()
+            if not root.is_dir():
+                continue
+
+            try:
+                for entry in root.rglob("*"):
+                    if len(results) >= max_results:
+                        break
+                    if not entry.is_file():
+                        continue
+
+                    file_name = entry.name
+                    file_stem = entry.stem
+
+                    tag_path = entry.parent / (file_name + ".tag")
+                    meta_path = entry.parent / (file_name + ".metadata")
+
+                    has_tag = tag_path.is_file()
+                    has_meta = meta_path.is_file()
+                    if not has_tag and not has_meta:
+                        continue
+
+                    tags: List[str] = []
+                    if has_tag:
+                        try:
+                            tags = read_tags_from_file(tag_path)
+                        except Exception:
+                            tags = []
+
+                    hash_value: Optional[str] = None
+                    urls: List[str] = []
+                    if has_meta:
+                        try:
+                            hash_value, _extra_tags, urls = _read_sidecar_metadata(meta_path)
+                        except Exception:
+                            pass
+
+                    if not match_all and query_tokens:
+                        url_text = " ".join(urls) if urls else ""
+                        search_text = " ".join([file_name.lower(), file_stem.lower().replace("_", " "), *tags, url_text])
+                        if hash_value:
+                            search_text += " " + hash_value.lower()
+                        if not all(token in search_text for token in query_tokens):
+                            continue
+
+                    try:
+                        size_bytes = int(entry.stat().st_size)
+                    except Exception:
+                        size_bytes = None
+
+                    ext = entry.suffix.lstrip(".")
+                    media_kind = self._infer_media_kind(ext)
+
+                    inst_label = str(inst_name or "").strip()
+                    if inst_label.lower() == "default":
+                        inst_label = ""
+
+                    store_label = f"local:{inst_label}" if inst_label else "local"
+                    tag_text = ", ".join(tags[:8]) if tags else ""
+
+                    metadata: Dict[str, Any] = {
+                        "store": store_label,
+                        "ext": ext,
+                        "size": size_bytes,
+                    }
+                    if inst_label:
+                        metadata["instance"] = inst_label
+                    if hash_value:
+                        metadata["hash"] = hash_value
+                    if urls:
+                        metadata["url"] = urls
+
+                    sr = SearchResult(
+                        table="local",
+                        title=file_name,
+                        path=str(entry),
+                        detail=store_label,
+                        annotations=[store_label],
+                        media_kind=media_kind,
+                        size_bytes=size_bytes,
+                        tag=set(tags),
+                        columns=[
+                            ("Title", file_name),
+                            ("Tag", tag_text),
+                            ("Store", store_label),
+                            ("Size", _format_size_safe(size_bytes)),
+                            ("Ext", ext),
+                        ],
+                        full_metadata=metadata,
+                    )
+                    sr.ext = ext
+                    sr.size = size_bytes
+                    if hash_value:
+                        sr.hash = hash_value
+                    results.append(sr)
+
+            except Exception:
+                continue
+
+        return results[:max_results]
+
     @staticmethod
     def _folder_name_from_pipe(pipe_obj: Any) -> str:
         metadata = getattr(pipe_obj, "metadata", None)

plugins/mpv/commands.py

@@ -1,5 +1,6 @@
 # pyright: reportUnusedFunction=false
 from typing import Any, Dict, Sequence, List, Optional
+import atexit
 import os
 import sys
 import json
@@ -21,6 +22,8 @@ from SYS.config import get_hydrus_access_key, get_hydrus_url, resolve_cookies_pa
 
 _NOTES_PREFETCH_INFLIGHT: set[str] = set()
 _NOTES_PREFETCH_LOCK = threading.Lock()
+_PREFETCH_THREADS: set[threading.Thread] = set()
+_PREFETCH_THREADS_LOCK = threading.Lock()
 _PLAYLIST_STORE_CACHE: Optional[Dict[str, Any]] = None
 _PLAYLIST_STORE_MTIME_NS: Optional[int] = None
 _SHA256_RE = re.compile(r"[0-9a-f]{64}")
@@ -347,21 +350,20 @@ def _get_latest_mpv_run_marker(log_db_path: str) -> Optional[tuple[str, str]]:
     try:
         import sqlite3
 
-        conn = sqlite3.connect(log_db_path, timeout=5.0)
+        with sqlite3.connect(log_db_path, timeout=5.0) as conn:
-        cur = conn.cursor()
+            cur = conn.cursor()
-        cur.execute(
+            cur.execute(
-            (
+                (
-                "SELECT timestamp, message FROM logs "
+                    "SELECT timestamp, message FROM logs "
-                "WHERE module = 'mpv' AND ("
+                    "WHERE module = 'mpv' AND ("
-                "message LIKE '[helper] version=% started ipc=%' "
+                    "message LIKE '[helper] version=% started ipc=%' "
-                "OR message LIKE 'medeia lua loaded version=%'"
+                    "OR message LIKE 'medeia lua loaded version=%'"
-                ") "
+                    ") "
-                "ORDER BY timestamp DESC LIMIT 1"
+                    "ORDER BY timestamp DESC LIMIT 1"
+                )
             )
-        )
+            row = cur.fetchone()
-        row = cur.fetchone()
+            cur.close()
-        cur.close()
-        conn.close()
         if row and row[0]:
             return str(row[0]), str(row[1] or "")
     except Exception:
@@ -381,26 +383,25 @@ def _get_mpv_logs_for_latest_run(
     try:
         import sqlite3
 
-        conn = sqlite3.connect(log_db_path, timeout=5.0)
+        with sqlite3.connect(log_db_path, timeout=5.0) as conn:
-        cur = conn.cursor()
+            cur = conn.cursor()
-        query = "SELECT timestamp, level, module, message FROM logs WHERE module = 'mpv'"
+            query = "SELECT timestamp, level, module, message FROM logs WHERE module = 'mpv'"
-        params: List[str] = []
+            params: List[str] = []
-        if marker_ts:
+            if marker_ts:
-            query += " AND timestamp >= ?"
+                query += " AND timestamp >= ?"
-            params.append(marker_ts)
+                params.append(marker_ts)
-        else:
+            else:
-            cutoff = (datetime.utcnow() - timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S")
+                cutoff = (datetime.utcnow() - timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S")
-            query += " AND timestamp >= ?"
+                query += " AND timestamp >= ?"
-            params.append(cutoff)
+                params.append(cutoff)
-        if log_filter_text:
+            if log_filter_text:
-            query += " AND LOWER(message) LIKE ?"
+                query += " AND LOWER(message) LIKE ?"
-            params.append(f"%{log_filter_text.lower()}%")
+                params.append(f"%{log_filter_text.lower()}%")
-        query += " ORDER BY timestamp DESC LIMIT ?"
+            query += " ORDER BY timestamp DESC LIMIT ?"
-        params.append(str(max(1, int(limit))))
+            params.append(str(max(1, int(limit))))
-        cur.execute(query, tuple(params))
+            cur.execute(query, tuple(params))
-        rows = cur.fetchall()
+            rows = cur.fetchall()
-        cur.close()
+            cur.close()
-        conn.close()
         rows.reverse()
         return marker_ts, marker_msg, rows
     except Exception:
@@ -849,11 +850,26 @@ def _prefetch_notes_async(
     thread = threading.Thread(
         target=_worker,
         name=f"mpv-notes-prefetch-{file_hash[:8]}",
-        daemon=True,
     )
+    with _PREFETCH_THREADS_LOCK:
+        _PREFETCH_THREADS.add(thread)
     thread.start()
 
 
+def _shutdown_prefetch_threads() -> None:
+    with _PREFETCH_THREADS_LOCK:
+        threads = list(_PREFETCH_THREADS)
+        _PREFETCH_THREADS.clear()
+    for t in threads:
+        try:
+            t.join(timeout=1.0)
+        except Exception:
+            pass
+
+
+atexit.register(_shutdown_prefetch_threads)
+
+
 def _schedule_notes_prefetch(items: Sequence[Any], config: Optional[Dict[str, Any]]) -> None:
     limit = _get_lyric_prefetch_limit(config)
     if limit <= 0:

plugins/mpv/format_probe.py

@@ -14,11 +14,6 @@ def _repo_root() -> Path:
     return package_dir.parent
 
 
-REPO_ROOT = _repo_root()
-if str(REPO_ROOT) not in sys.path:
-    sys.path.insert(0, str(REPO_ROOT))
-
-
 def main(argv: list[str] | None = None) -> int:
     args = list(sys.argv[1:] if argv is None else argv)
     if not args:
@@ -35,10 +30,19 @@ def main(argv: list[str] | None = None) -> int:
     url = str(args[0] or "").strip()
     captured_stdout = io.StringIO()
     captured_stderr = io.StringIO()
-    with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr):
-        from plugins.mpv.pipeline_helper import _run_op
 
-        payload = _run_op("ytdlp-formats", {"url": url})
+    _root_str = str(_repo_root())
+    _path_added = _root_str not in sys.path
+    if _path_added:
+        sys.path.insert(0, _root_str)
+    try:
+        with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr):
+            from plugins.mpv.pipeline_helper import _run_op
+
+            payload = _run_op("ytdlp-formats", {"url": url})
+    finally:
+        if _path_added:
+            sys.path.remove(_root_str)
 
     noisy_stdout = captured_stdout.getvalue().strip()
     noisy_stderr = captured_stderr.getvalue().strip()

plugins/mpv/mpv_ipc.py

@@ -7,6 +7,7 @@ This is the central hub for all Python-mpv IPC communication. The Lua script
 should use the Python CLI, which uses this module to manage mpv connections.
 """
 
+import atexit
 import ctypes
 import json
 import os
@@ -901,6 +902,7 @@ class MPVIPCClient:
         self.is_windows = platform.system() == "Windows"
         self.silent = bool(silent)
         self._recv_buffer: bytes = b""
+        atexit.register(self.disconnect)
 
     def _write_payload(self, payload: str) -> None:
         if not self.sock:
@@ -1229,10 +1231,6 @@ class MPVIPCClient:
             self.sock = None
         self._recv_buffer = b""
 
-    def __del__(self) -> None:
-        """Cleanup on object destruction."""
-        self.disconnect()
-
     def __enter__(self):
         """Context manager entry."""
         self.connect()

plugins/mpv/pipeline_helper.py

@@ -61,17 +61,22 @@ def _runtime_config_root() -> Path:
 
 
 # Make repo-local packages importable even when mpv starts us from another cwd.
-_ROOT = str(_repo_root())
+_root_str = str(_repo_root())
-if _ROOT not in sys.path:
+_path_added = _root_str not in sys.path
-    sys.path.insert(0, _ROOT)
+if _path_added:
-
+    sys.path.insert(0, _root_str)
-from plugins.mpv.mpv_ipc import MPVIPCClient, _windows_kill_pids, _windows_hidden_subprocess_kwargs, _windows_list_mpv_pids  # noqa: E402
+try:
-from SYS.config import load_config, reload_config  # noqa: E402
+    from plugins.mpv.mpv_ipc import MPVIPCClient, _windows_kill_pids, _windows_hidden_subprocess_kwargs, _windows_list_mpv_pids  # noqa: E402
-from SYS.logger import set_debug, debug, set_thread_stream  # noqa: E402
+    from SYS.config import load_config, reload_config  # noqa: E402
-from SYS.repl_queue import enqueue_repl_command, repl_state_is_alive  # noqa: E402
+    from SYS.logger import set_debug, debug, set_thread_stream  # noqa: E402
-from SYS.utils import format_bytes  # noqa: E402
+    from SYS.repl_queue import enqueue_repl_command, repl_state_is_alive  # noqa: E402
-from PluginCore.registry import get_plugin, get_plugin_class  # noqa: E402
+    from SYS.utils import format_bytes  # noqa: E402
-from plugins.ytdlp.tooling import get_display_format_id, get_selection_format_id  # noqa: E402
+    from PluginCore.registry import get_plugin, get_plugin_class  # noqa: E402
+    from plugins.ytdlp.tooling import get_display_format_id, get_selection_format_id  # noqa: E402
+finally:
+    if _path_added:
+        sys.path.remove(_root_str)
+    del _root_str, _path_added
 
 REQUEST_PROP = "user-data/medeia-pipeline-request"
 RESPONSE_PROP = "user-data/medeia-pipeline-response"

plugins/playwright/__init__.py

@@ -7,7 +7,10 @@ import `plugins.playwright` even when Playwright itself is not installed.
 
 from __future__ import annotations
 
+from .runtime import USER_AGENT
+
 __all__ = [
+    "USER_AGENT",
     "PlaywrightTimeoutError",
     "PlaywrightTool",
     "PlaywrightDefaults",

plugins/playwright/runtime.py

@@ -65,15 +65,18 @@ def _find_filename_from_cd(cd: str) -> Optional[str]:
     return None
 
 
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/120.0.0.0 Safari/537.36"
+)
+
+
 @dataclass(slots=True)
 class PlaywrightDefaults:
     browser: str = "chromium"  # chromium|firefox|webkit
     headless: bool = True
-    user_agent: str = (
+    user_agent: str = USER_AGENT
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/120.0.0.0 Safari/537.36"
-    )
     viewport_width: int = 1920
     viewport_height: int = 1080
     navigation_timeout_ms: int = 90_000

plugins/soulseek/__init__.py

@@ -243,9 +243,11 @@ class _LineFilterStream(io.TextIOBase):
 def _suppress_aioslsk_noise() -> Any:
     """Temporarily suppress known aioslsk noise printed to stdout/stderr.
 
-    Opt out by setting DOWNLOW_SOULSEEK_VERBOSE=1.
+    Opt out by setting DOWNLOAD_SOULSEEK_VERBOSE=1.
+
+    Note: The old env var name DOWNLOW_SOULSEEK_VERBOSE (typo) is deprecated.
     """
-    if os.environ.get("DOWNLOW_SOULSEEK_VERBOSE"):
+    if os.environ.get("DOWNLOAD_SOULSEEK_VERBOSE") or os.environ.get("DOWNLOW_SOULSEEK_VERBOSE"):
         yield
         return
 
@@ -404,7 +406,7 @@ class Soulseek(Provider):
             )
 
         except RuntimeError:
-            # If we're already inside an event loop (e.g., TUI), fall back to a
+            # If we're already inside an event loop, fall back to a
             # dedicated loop in this thread.
             loop = asyncio.new_event_loop()
             try:
@@ -568,6 +570,21 @@ class Soulseek(Provider):
                                     timeout=9.0,
                                     limit=limit)
             )
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            try:
+                asyncio.set_event_loop(loop)
+                flat_results, search_summary = loop.run_until_complete(
+                    self.perform_search(query, timeout=9.0, limit=limit)
+                )
+            finally:
+                try:
+                    loop.run_until_complete(loop.shutdown_asyncgens())
+                except Exception:
+                    pass
+                loop.close()
+
+        try:
             if not flat_results:
                 return []
 

plugins/telegram/__init__.py

@@ -319,7 +319,7 @@ class Telegram(Provider):
     def _run_async_blocking(self, coro):
         """Run an awaitable to completion using a fresh event loop.
 
-        If an event loop is already running in this thread (common in REPL/TUI),
+        If an event loop is already running in this thread (common in REPL),
         runs the coroutine in a worker thread with its own loop.
         """
         result: Dict[str,
@@ -358,6 +358,13 @@ class Telegram(Provider):
                 except Exception:
                     pass
             finally:
+                try:
+                    try:
+                        loop.run_until_complete(loop.shutdown_default_executor())
+                    except Exception:
+                        pass
+                except Exception:
+                    pass
                 try:
                     loop.close()
                 except Exception:
@@ -559,6 +566,10 @@ class Telegram(Provider):
                     asyncio.set_event_loop(loop)
                     loop.run_until_complete(_auth_async())
                 finally:
+                    try:
+                        loop.run_until_complete(loop.shutdown_default_executor())
+                    except Exception:
+                        pass
                     try:
                         loop.close()
                     except Exception:

plugins/ytdlp/__init__.py

@@ -510,6 +510,14 @@ class ytdlp(TablePluginMixin, Provider):
     SEARCH_QUERY_KEYS = ("search", "q")
     SUPPORTED_CMDLETS = frozenset({"download-file", "search-file"})
 
+    QUERY_ARG_CHOICES = {
+        "format": ["audio", "1080", "720", "480", "360", "best", "worst", "bestvideo+bestaudio"],
+        "clip": ["0:00-1:00"],
+        "audio": ["true", "false", "1", "0"],
+        "item": ["1"],
+    }
+    INLINE_QUERY_FIELD_CHOICES = QUERY_ARG_CHOICES
+
     @staticmethod
     def config_schema() -> List[Dict[str, Any]]:
         return _ytdlp_config_schema()

pyproject.toml

@@ -9,7 +9,7 @@ description = "Comprehensive media management and search platform with support f
 requires-python = ">=3.9,<3.14"
 license = {text = "MIT"}
 authors = [
-    {name = "Your Name", email = "your.email@example.com"}
+    {name = "Nose", email = "goyimnose@nothing.blah"}
 ]
 keywords = ["media", "search", "management", "hydrus", "download", "cli", "tui"]
 classifiers = [
@@ -31,6 +31,7 @@ classifiers = [
 dependencies = [
     # Core CLI and TUI frameworks
     "typer>=0.9.0",
+    "rich>=13.7.0",
     "prompt-toolkit>=3.0.0",
     "textual>=0.30.0",
 
@@ -41,12 +42,14 @@ dependencies = [
     "charset-normalizer>=3.2.0",
     "certifi>=2024.12.0",
     "httpx>=0.25.0",
+    "internetarchive>=4.1.0",
 
     # Document and data handling
     "pypdf>=3.0.0",
     "mutagen>=1.46.0",
     "cbor2>=4.0",
     "zstandard>=0.23.0",
+    "pypandoc-binary",
 
     # Image and media support
     "Pillow>=10.0.0",
@@ -109,10 +112,10 @@ mm = "scripts.cli_entry:main"
 medeia = "scripts.cli_entry:main"
 
 [project.urls]
-Homepage = "https://github.com/yourusername/medeia-macina"
+Homepage = "https://code.glowers.club/goyimnose/Medios-Macina"
-Documentation = "https://medeia-macina.readthedocs.io"
+Documentation = "https://code.glowers.club/goyimnose/Medios-Macina"
-Repository = "https://github.com/yourusername/medeia-macina.git"
+Repository = "https://code.glowers.club/goyimnose/Medios-Macina.git"
-Issues = "https://github.com/yourusername/medeia-macina/issues"
+Issues = "https://code.glowers.club/goyimnose/Medios-Macina/issues"
 
 [tool.setuptools]
 packages = [
@@ -123,7 +126,7 @@ packages = [
     "PluginCore",
     "SYS",
 ]
-package-dir = {"" = ".."}
+package-dir = {"" = "."}
 
 [tool.setuptools.package-data]
 scripts = ["*.py"]

readme.md

@@ -41,9 +41,7 @@ See [plugins/README.md](plugins/README.md) for plugin packaging and discovery de
 
 ## Configuration
 
-The old interactive TUI config editor has been discontinued.
+Use `.config` from inside the CLI:
-
-Use `.config` from inside the CLI instead:
 
 - `.config` opens the root configuration table.
 - `@N` drills into a selected section.

scripts/_debug_token_windows_path.py

@@ -0,0 +1,10 @@
+from CLI import MedeiaLexer
+
+
+class DummyDocument:
+    def __init__(self, lines):
+        self.lines = list(lines)
+
+lexer = MedeiaLexer()
+doc = DummyDocument([r'C:\path\to\file'])
+print(lexer.lex_document(doc)(0))

scripts/_run_cli_parsing_manual.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import sys
+import traceback
+
+sys.path.insert(0, r'C:\Forgejo\Medios-Macina')
+from tests.test_cli_parsing import *
+
+failed = 0
+tests = [(n, o) for n, o in globals().items() if n.startswith('test_') and callable(o)]
+for n, o in tests:
+    try:
+        o()
+        print(n, 'OK')
+    except Exception as e:
+        print(n, 'FAIL', e)
+        traceback.print_exc()
+        failed += 1
+
+if failed:
+    sys.exit(1)
+print('All tests passed')

scripts/check_imports.py

@@ -0,0 +1,10 @@
+import importlib
+import sys
+import traceback
+
+try:
+    importlib.import_module("CLI")
+    print("CLI imported OK")
+except Exception:
+    traceback.print_exc()
+    sys.exit(1)

scripts/check_indentation.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+p = Path('Store/registry.py')
+lines = p.read_text(encoding='utf-8').splitlines()
+stack = [0]
+for i, line in enumerate(lines, start=1):
+    if not line.strip():
+        continue
+    leading = len(line) - len(line.lstrip(' '))
+    if leading > stack[-1]:
+        # increased indent -> push
+        stack.append(leading)
+    else:
+        # dedent: must match an existing level
+        if leading != stack[-1]:
+            if leading in stack:
+                # pop until match
+                while stack and stack[-1] != leading:
+                    stack.pop()
+            else:
+                print(f"Line {i}: inconsistent indent {leading}, stack levels {stack}")
+                print(repr(line))
+                break
+print('Done')

scripts/check_pattern.py

@@ -0,0 +1,19 @@
+import re
+from pathlib import Path
+p = Path(r'c:\Forgejo\Medios-Macina\CLI.py')
+s = p.read_text(encoding='utf-8')
+pattern = re.compile(r'(?s)if False:\s*class _OldPipelineExecutor:.*?from rich\\.markdown import Markdown\\s*')
+m = pattern.search(s)
+print('found', bool(m))
+if m:
+    print('start', m.start(), 'end', m.end())
+    print('snippet:', s[m.start():m.start()+120])
+else:
+    # print a slice around the if False for debugging
+    i = s.find('if False:')
+    print('if False index', i)
+    print('around if False:', s[max(0,i-50):i+200])
+    j = s.find('from rich.markdown import Markdown', i)
+    print('next from rich index after if False', j)
+    if j!=-1:
+        print('around that:', s[j-50:j+80])

scripts/check_store.py

@@ -0,0 +1,12 @@
+import traceback
+
+try:
+    from PluginCore.backend_registry import BackendRegistry
+    s = BackendRegistry(config={}, suppress_debug=True)
+    print('INSTANCE TYPE:', type(s))
+    print('HAS is_available:', hasattr(s, 'is_available'))
+    if hasattr(s, 'is_available'):
+        print('is_available callable:', callable(getattr(s, 'is_available')))
+    print('DIR:', sorted([n for n in dir(s) if not n.startswith('__')]))
+except Exception:
+    traceback.print_exc()

scripts/check_try_balance.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+p=Path('SYS/pipeline.py')
+s=p.read_text(encoding='utf-8')
+lines=s.splitlines()
+stack=[]
+for i,l in enumerate(lines,1):
+    stripped=l.strip()
+    # Skip commented lines
+    if stripped.startswith('#'):
+        continue
+    # compute indent as leading spaces (tabs are converted)
+    indent = len(l) - len(l.lstrip(' '))
+    if stripped.startswith('try:'):
+        stack.append((indent, i))
+    if stripped.startswith('except ') or stripped=='except:' or stripped.startswith('finally:'):
+        # find the most recent try with same indent
+        for idx in range(len(stack)-1, -1, -1):
+            if stack[idx][0] == indent:
+                stack.pop(idx)
+                break
+        else:
+            # no matching try at same indent
+            print(f"Found {stripped.split()[0]} at line {i} with no matching try at same indent")
+
+print('Unmatched try count', len(stack))
+if stack:
+    print('Unmatched try positions (indent, line):', stack)
+    for indent, lineno in stack:
+        start = max(1, lineno - 10)
+        end = min(len(lines), lineno + 10)
+        print(f"Context around line {lineno}:")
+        for i in range(start, end + 1):
+            print(f"{i:5d}: {lines[i-1]}")
+else:
+    print("All try statements appear matched")

scripts/cli_entry.py

@@ -8,7 +8,7 @@ running from a development checkout (by importing the top-level
 
 from __future__ import annotations
 
-from typing import Optional, List, Tuple
+from typing import Optional, List
 import importlib
 import importlib.util
 import os
@@ -64,91 +64,32 @@ def _ensure_repo_root_on_sys_path(pkg_file: Optional[Path] = None) -> Optional[P
     return None
 
 
-def _parse_mode_and_strip_args(args: List[str]) -> Tuple[Optional[str], List[str]]:
+def _strip_mode_args(args: List[str]) -> List[str]:
-    """Parse --gui/--cli/--mode flags and return (mode, cleaned_args).
+    """Strip --gui/--cli/--mode flags from argument list.
 
-    The function removes any mode flags from the argument list so the selected
+    The GUI/TUI mode has been discontinued. Any mode flags are silently
-    runner can receive the remaining arguments untouched.
+    consumed so they don't interfere with remaining argument parsing.
-
-    Supported forms:
-      --gui, -g, --gui=true
-      --cli, -c, --cli=true
-      --mode=gui|cli
-      --mode gui|cli
-
-    Raises ValueError on conflicting or invalid flags.
     """
-    mode: Optional[str] = None
     out: List[str] = []
     i = 0
     while i < len(args):
         a = args[i]
         la = a.lower()
-
+        if la in ("--gui", "-g", "--cli", "-c"):
-        # --gui / -g
-        if la in ("--gui", "-g"):
-            if mode and mode != "gui":
-                raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'")
-            mode = "gui"
             i += 1
             continue
-        if la.startswith("--gui="):
+        if la.startswith("--gui=") or la.startswith("--cli="):
-            val = la.split("=", 1)[1]
-            if val and val not in ("0", "false", "no", "off"):
-                if mode and mode != "gui":
-                    raise ValueError(
-                        "Conflicting mode flags: found both 'gui' and 'cli'"
-                    )
-                mode = "gui"
             i += 1
             continue
-
-        # --cli / -c
-        if la in ("--cli", "-c"):
-            if mode and mode != "cli":
-                raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'")
-            mode = "cli"
-            i += 1
-            continue
-        if la.startswith("--cli="):
-            val = la.split("=", 1)[1]
-            if val and val not in ("0", "false", "no", "off"):
-                if mode and mode != "cli":
-                    raise ValueError(
-                        "Conflicting mode flags: found both 'gui' and 'cli'"
-                    )
-                mode = "cli"
-            i += 1
-            continue
-
-        # --mode
         if la.startswith("--mode="):
-            val = la.split("=", 1)[1]
-            val = val.lower()
-            if val not in ("gui", "cli"):
-                raise ValueError("--mode must be 'gui' or 'cli'")
-            if mode and mode != val:
-                raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'")
-            mode = val
             i += 1
             continue
         if la == "--mode":
-            if i + 1 >= len(args):
-                raise ValueError("--mode requires a value ('gui' or 'cli')")
-            val = args[i + 1].lower()
-            if val not in ("gui", "cli"):
-                raise ValueError("--mode must be 'gui' or 'cli'")
-            if mode and mode != val:
-                raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'")
-            mode = val
             i += 2
             continue
-
-        # Not a mode flag; keep it
         out.append(a)
         i += 1
-
+    return out
-    return mode, out
 
 
 def _run_cli(clean_args: List[str]) -> int:
@@ -221,15 +162,6 @@ def _run_cli(clean_args: List[str]) -> int:
         return int(getattr(exc, "code", 0) or 0)
 
 
-def _run_gui(clean_args: List[str]) -> int:
-    """Report that the discontinued GUI/TUI mode is no longer available."""
-    _ = clean_args
-    print(
-        "Error: GUI/TUI mode has been discontinued and is no longer available.",
-        file=sys.stderr,
-    )
-    return 2
-
 
 def main(argv: Optional[List[str]] = None) -> int:
     """Entry point for console_scripts.
@@ -239,11 +171,7 @@ def main(argv: Optional[List[str]] = None) -> int:
     """
     args = list(argv) if argv is not None else list(sys.argv[1:])
 
-    try:
+    clean_args = _strip_mode_args(args)
-        mode, clean_args = _parse_mode_and_strip_args(args)
-    except ValueError as exc:
-        print(f"Error parsing mode flags: {exc}", file=sys.stderr)
-        return 2
 
     # Early environment sanity check to detect urllib3/urllib3-future conflicts.
     # When a broken urllib3 is detected we print an actionable message and
@@ -262,10 +190,6 @@ def main(argv: Optional[List[str]] = None) -> int:
         # startup; we'll continue and let normal import errors surface.
         pass
 
-    # If GUI requested, delegate directly (GUI may decide to honor any args itself)
-    if mode == "gui":
-        return _run_gui(clean_args)
-
     # Support quoting a pipeline (or even a single full command) on the command line.
     #
     # - If the user provides a single argument that contains a pipe character,

scripts/debug_import_vimm.py

@@ -0,0 +1,10 @@
+import importlib
+import traceback
+
+try:
+    m = importlib.import_module('Provider.vimm')
+    print('Imported', m)
+    print('Vimm class:', getattr(m, 'Vimm', None))
+except Exception as e:
+    print('Import failed:', e)
+    traceback.print_exc()

scripts/find_big_indent.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+p = Path('Store/registry.py')
+for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
+    leading = len(line) - len(line.lstrip(' '))
+    if leading > 20:
+        print(i, leading, repr(line))

scripts/format_tracked.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Format all tracked Python files using the repo's YAPF style.
+
+This script intentionally formats only files tracked by git to avoid touching
+files that are ignored (e.g., venv, site-packages).
+"""
+import subprocess
+import sys
+
+try:
+    out = subprocess.check_output(["git", "ls-files", "*.py"]).decode("utf-8")
+except subprocess.CalledProcessError as exc:
+    print("Failed to get tracked files from git:", exc, file=sys.stderr)
+    sys.exit(1)
+
+files = [f for f in (line.strip() for line in out.splitlines()) if f]
+print(f"Formatting {len(files)} tracked python files...")
+if not files:
+    print("No tracked python files found.")
+    sys.exit(0)
+
+for path in files:
+    print(path)
+    subprocess.run([sys.executable, "-m", "yapf", "-i", "--style", ".style.yapf", path], check=False)
+
+print("Done")

scripts/indent_check.py

@@ -0,0 +1,11 @@
+import pathlib
+p = pathlib.Path('Store/registry.py')
+with p.open('r', encoding='utf-8') as f:
+    lines = f.readlines()
+for i in range(312, 328):
+    if i-1 < len(lines):
+        line = lines[i-1]
+        leading = line[:len(line)-len(line.lstrip('\t '))]
+        print(f"{i}: {repr(line.rstrip())} | leading={repr(leading)} len={len(leading)} chars={[ord(c) for c in leading]}")
+    else:
+        print(f"{i}: <EOF>")

scripts/indent_stats.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+p = Path('Store/registry.py')
+counts = {}
+for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
+    if not line.strip():
+        continue
+    leading = len(line) - len(line.lstrip(' '))
+    counts[leading] = counts.get(leading, 0) + 1
+for k in sorted(counts.keys()):
+    print(k, counts[k])

scripts/list_providers.py

@@ -0,0 +1,3 @@
+from PluginCore.registry import list_plugins
+
+print('All plugins:', list_plugins())

scripts/requirements.txt

@@ -14,6 +14,7 @@ charset-normalizer>=3.2.0
 certifi>=2024.12.0
 # Optional Telegram support installs telethon>=1.36.0 when [provider=telegram] is configured.
 internetarchive>=4.1.0
+yt-dlp-ejs
 
 # Document and data handling
 pypdf>=3.0.0
@@ -44,7 +45,8 @@ tqdm>=4.66.0
 
 # Browser automation (for web scraping if needed)
 playwright>=1.40.0
+paramiko>=3.5.0
+scp>=0.15.0
 
 # Development and utilities
 python-dateutil>=2.8.0
-