""" """

from __future__ import annotations

import base64
import hashlib
import json
import re
import shutil
import sys
import tempfile
import time
from collections.abc import Iterable as IterableABC
from functools import lru_cache
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse

from SYS.logger import log, debug, debug_panel
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from SYS import models
from SYS import pipeline as pipeline_context
from SYS.item_accessors import get_field as _item_accessor_get_field
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.result_publication import publish_result_table
from SYS.result_table import Table
from SYS.rich_display import stderr_console as get_stderr_console
from SYS.cmdlet_spec import Cmdlet, CmdletArg, QueryArg, SharedArgs, parse_cmdlet_args
from rich.prompt import Confirm
from contextlib import AbstractContextManager, nullcontext


# Tag groups cache (loaded from JSON config file)
_TAG_GROUPS_CACHE: Optional[Dict[str, List[str]]] = None
_TAG_GROUPS_MTIME: Optional[float] = None

# Path to tag groups configuration (set by caller or lazily discovered)
TAG_GROUPS_PATH: Optional[Path] = None


def set_tag_groups_path(path: Path) -> None:
    """Set the path to the tag groups JSON file."""
    global TAG_GROUPS_PATH
    TAG_GROUPS_PATH = path


def resolve_target_dir(
    parsed: Dict[str, Any],
    config: Dict[str, Any],
    *,
    handle_creations: bool = True
) -> Optional[Path]:
    """Resolve a target directory from -path, -storage, or config fallback.

    Args:
        parsed: Parsed cmdlet arguments dict.
        config: System configuration dict.
        handle_creations: Whether to create the directory if it doesn't exist.

    Returns:
        Path to the resolved directory, or None if invalid.
    """
    # Priority 1: Explicit -path
    target = parsed.get("path")
    if target:
        try:
            from SYS.config import resolve_path_alias
            from SYS.utils import expand_path

            raw_target = str(target or "").strip()
            aliased = resolve_path_alias(config, raw_target)
            if raw_target.startswith("$") and aliased is None:
                log(
                    f"Unknown path alias {raw_target}. Set it with .config path_aliases.<name> <path>",
                    file=sys.stderr,
                )
                return None

            p = aliased if aliased is not None else expand_path(raw_target)
            if handle_creations:
                p.mkdir(parents=True, exist_ok=True)
            return p
        except Exception as e:
            log(f"Cannot use target path {target}: {e}", file=sys.stderr)
            return None

    # Priority 2: --storage flag
    storage_val = parsed.get("storage")
    if storage_val:
        try:
            return SharedArgs.resolve_storage(storage_val)
        except Exception as e:
            log(f"Invalid storage location: {e}", file=sys.stderr)
            return None

    # Priority 3: Config fallback via single source of truth
    try:
        from SYS.config import resolve_output_dir
        out_dir = resolve_output_dir(config)
        if handle_creations:
            out_dir.mkdir(parents=True, exist_ok=True)
        return out_dir
    except Exception:
        import tempfile
        p = Path(tempfile.gettempdir()) / "Medios-Macina"
        if handle_creations:
            p.mkdir(parents=True, exist_ok=True)
        return p


def coerce_to_path(value: Any) -> Path:
    """Extract a Path from common provider result shapes (Path, str, dict, object)."""
    if isinstance(value, Path):
        return value
    if isinstance(value, str):
        return Path(value)

    # Try attribute
    p = getattr(value, "path", None)
    if p:
        return Path(str(p))

    # Try dict
    if isinstance(value, dict):
        p = value.get("path")
        if p:
            return Path(str(p))

    raise ValueError(f"Cannot coerce {type(value).__name__} to Path (missing 'path' field)")



def resolve_media_kind_by_extension(path: Path) -> str:
    """Resolve media kind (audio, video, image, document, other) from file extension."""
    if not isinstance(path, Path):
        try:
            path = Path(str(path))
        except Exception:
            return "other"
            
    suffix = path.suffix.lower()
    if suffix in {".mp3",
                  ".flac",
                  ".wav",
                  ".m4a",
                  ".aac",
                  ".ogg",
                  ".opus",
                  ".wma",
                  ".mka"}:
        return "audio"
    if suffix in {
            ".mp4",
            ".mkv",
            ".webm",
            ".mov",
            ".avi",
            ".flv",
            ".mpg",
            ".mpeg",
            ".ts",
            ".m4v",
            ".wmv",
    }:
        return "video"
    if suffix in {".jpg",
                  ".jpeg",
                  ".png",
                  ".gif",
                  ".webp",
                  ".bmp",
                  ".tiff"}:
        return "image"
    if suffix in {".pdf",
                  ".epub",
                  ".txt",
                  ".mobi",
                  ".azw3",
                  ".cbz",
                  ".cbr",
                  ".doc",
                  ".docx"}:
        return "document"
    return "other"


def build_pipeline_preview(raw_urls: Sequence[str], piped_items: Sequence[Any]) -> List[str]:
    """Construct a short preview list for pipeline/cmdlet progress UI."""
    preview: List[str] = []

    # 1. Add raw URLs
    try:
        for u in (raw_urls or [])[:3]:
            if u:
                preview.append(str(u))
    except Exception:
        pass

    # 2. Add titles from piped items
    if len(preview) < 5:
        try:
            for item in (piped_items or [])[:5]:
                if len(preview) >= 5:
                    break
                title = get_field(item, "title") or get_field(item, "target") or "Piped item"
                preview.append(str(title))
        except Exception:
            pass

    # 3. Handle empty case
    if not preview:
        total = len(raw_urls or []) + len(piped_items or [])
        if total:
            preview.append(f"Processing {total} item(s)...")

    return preview


@lru_cache(maxsize=4096)
def _normalize_hash_cached(hash_hex: str) -> Optional[str]:
    text = hash_hex.strip().lower()
    if not text:
        return None
    if len(text) != 64:
        return None
    if not all(ch in "0123456789abcdef" for ch in text):
        return None
    return text


def normalize_hash(hash_hex: Optional[str]) -> Optional[str]:
    """Normalize a hash string to lowercase, or return None if invalid.

    Args:
            hash_hex: String that should be a hex hash

    Returns:
            Lowercase hash string, or None if input is not a string or is empty
    """
    if not isinstance(hash_hex, str):
        return None
    return _normalize_hash_cached(hash_hex)


def resolve_hash_for_cmdlet(
    raw_hash: Optional[str],
    raw_path: Optional[str],
    override_hash: Optional[str],
) -> Optional[str]:
    """Resolve a file hash for note/tag/file cmdlets.

    Shared implementation used by add-note, delete-note, get-note, and similar
    cmdlets that need to identify a file by its SHA-256 hash.

    Resolution order:
    1. ``override_hash`` — explicit hash provided via *-query* (highest priority)
    2. ``raw_hash`` — positional hash argument
    3. ``raw_path`` stem — if the filename stem is a 64-char hex string it is
       treated directly as the hash (Hydrus-style naming convention)
    4. SHA-256 computed from the file at ``raw_path``

    Args:
        raw_hash: Hash string from positional argument.
        raw_path: Filesystem path to the file (may be None).
        override_hash: Hash extracted from *-query* (takes precedence).

    Returns:
        Normalised 64-char lowercase hex hash, or ``None`` if unresolvable.
    """
    resolved = normalize_hash(override_hash) if override_hash else normalize_hash(raw_hash)
    if resolved:
        return resolved
    if raw_path:
        try:
            p = Path(str(raw_path))
            stem = p.stem
            if len(stem) == 64 and all(c in "0123456789abcdef" for c in stem.lower()):
                return stem.lower()
            if p.exists() and p.is_file():
                from SYS.utils import sha256_file as _sha256_file
                return _sha256_file(p)
        except Exception:
            return None
    return None


def resolve_item_store_hash(
    item: Any,
    *,
    override_store: Optional[str] = None,
    override_hash: Optional[str] = None,
    hash_field: str = "hash",
    store_field: str = "store",
    path_fields: Sequence[str] = ("path", "target"),
) -> Tuple[str, Optional[str]]:
    """Resolve store name and normalized hash from a result item."""
    store_name = str(override_store or get_field(item, store_field) or "").strip()
    raw_hash = get_field(item, hash_field)

    raw_path = None
    for field_name in path_fields:
        candidate = get_field(item, field_name)
        if candidate:
            raw_path = candidate
            break

    resolved_hash = resolve_hash_for_cmdlet(
        str(raw_hash) if raw_hash else None,
        str(raw_path) if raw_path else None,
        str(override_hash) if override_hash else None,
    )
    return store_name, resolved_hash


def get_store_backend(
    config: Optional[Dict[str, Any]],
    store_name: Optional[str],
    *,
    store_registry: Any = None,
    suppress_debug: bool = False,
) -> Tuple[Optional[Any], Any, Optional[Exception]]:
    """Resolve a store backend, optionally reusing an existing registry.

    Returns a tuple of ``(backend, store_registry, exc)`` so callers can keep
    their command-specific error messages while avoiding repeated registry setup
    and ``store[name]`` boilerplate.
    """
    registry = store_registry
    if registry is None:
        try:
            from PluginCore.backend_registry import BackendRegistry

            registry = BackendRegistry(config or {}, suppress_debug=suppress_debug)
        except Exception as exc:
            return None, None, exc

    backend_name = str(store_name or "").strip()
    if not backend_name:
        return None, registry, KeyError("Missing store name")

    try:
        return registry[backend_name], registry, None
    except Exception as exc:
        return None, registry, exc


def get_preferred_store_backend(
    config: Optional[Dict[str, Any]],
    store_name: Optional[str],
    *,
    store_registry: Any = None,
    suppress_debug: bool = True,
) -> Tuple[Optional[Any], Any, Optional[Exception]]:
    """Prefer a targeted backend instance before falling back to registry lookup."""
    direct_exc: Optional[Exception] = None
    try:
        from PluginCore.backend_registry import get_backend_instance

        backend = get_backend_instance(
            config or {},
            str(store_name or ""),
            suppress_debug=suppress_debug,
        )
        if backend is not None:
            return backend, store_registry, None
    except Exception as exc:
        direct_exc = exc

    backend, registry, lookup_exc = get_store_backend(
        config,
        store_name,
        store_registry=store_registry,
        suppress_debug=suppress_debug,
    )
    if backend is not None:
        return backend, registry, None
    return None, registry, direct_exc or lookup_exc


def coalesce_hash_value_pairs(
    pairs: Sequence[Tuple[str, Sequence[str]]],
) -> List[Tuple[str, List[str]]]:
    """Merge duplicate hash/value pairs while preserving first-seen value order."""
    merged: Dict[str, List[str]] = {}
    for hash_value, values in pairs:
        normalized_hash = str(hash_value or "").strip()
        if not normalized_hash:
            continue
        bucket = merged.setdefault(normalized_hash, [])
        seen = set(bucket)
        for value in values or []:
            text = str(value or "").strip()
            if not text or text in seen:
                continue
            seen.add(text)
            bucket.append(text)
    return [(hash_value, items) for hash_value, items in merged.items() if items]


def run_store_hash_value_batches(
    config: Optional[Dict[str, Any]],
    batch: Dict[str, List[Tuple[str, Sequence[str]]]],
    *,
    bulk_method_name: str,
    single_method_name: str,
    store_registry: Any = None,
    suppress_debug: bool = False,
    pass_config_to_bulk: bool = True,
    pass_config_to_single: bool = True,
) -> Tuple[Any, List[Tuple[str, int, int]]]:
    """Dispatch grouped hash/value batches across stores.

    Returns ``(store_registry, stats)`` where ``stats`` contains
    ``(store_name, item_count, value_count)`` for each dispatched store.
    Missing stores are skipped so callers can preserve existing warning behavior.
    """
    registry = store_registry
    stats: List[Tuple[str, int, int]] = []
    for store_name, pairs in batch.items():
        backend, registry, _exc = get_store_backend(
            config,
            store_name,
            store_registry=registry,
            suppress_debug=suppress_debug,
        )
        if backend is None:
            continue

        bulk_pairs = coalesce_hash_value_pairs(pairs)
        if not bulk_pairs:
            continue

        bulk_fn = getattr(backend, bulk_method_name, None)
        if callable(bulk_fn):
            if pass_config_to_bulk:
                bulk_fn(bulk_pairs, config=config)
            else:
                bulk_fn(bulk_pairs)
        else:
            single_fn = getattr(backend, single_method_name)
            for hash_value, values in bulk_pairs:
                if pass_config_to_single:
                    single_fn(hash_value, values, config=config)
                else:
                    single_fn(hash_value, values)

        stats.append(
            (
                store_name,
                len(bulk_pairs),
                sum(len(values or []) for _hash_value, values in bulk_pairs),
            )
        )

    return registry, stats


def run_store_note_batches(
    config: Optional[Dict[str, Any]],
    batch: Dict[str, List[Tuple[str, str, str]]],
    *,
    store_registry: Any = None,
    suppress_debug: bool = False,
    on_store_error: Optional[Callable[[str, Exception], None]] = None,
    on_unsupported_store: Optional[Callable[[str], None]] = None,
    on_item_error: Optional[Callable[[str, str, str, Exception], None]] = None,
) -> Tuple[Any, int]:
    """Dispatch grouped note writes across stores while preserving item-level errors."""
    registry = store_registry
    success_count = 0
    for store_name, items in batch.items():
        backend, registry, exc = get_store_backend(
            config,
            store_name,
            store_registry=registry,
            suppress_debug=suppress_debug,
        )
        if backend is None:
            if on_store_error is not None and exc is not None:
                on_store_error(store_name, exc)
            continue
        supports_note_capability = getattr(backend, "supports_note_association", None)
        if supports_note_capability is None:
            supports_note_capability = hasattr(backend, "set_note")
        if not bool(supports_note_capability):
            if on_unsupported_store is not None:
                on_unsupported_store(store_name)
            continue
        if not hasattr(backend, "set_note"):
            if on_unsupported_store is not None:
                on_unsupported_store(store_name)
            continue

        for hash_value, note_name, note_text in items:
            try:
                if backend.set_note(hash_value, note_name, note_text, config=config):
                    success_count += 1
            except Exception as item_exc:
                if on_item_error is not None:
                    on_item_error(store_name, hash_value, note_name, item_exc)

    return registry, success_count


def collect_store_hash_value_batch(
    items: Sequence[Any],
    *,
    store_registry: Any,
    value_resolver: Callable[[Any], Optional[Sequence[str]]],
    override_hash: Optional[str] = None,
    override_store: Optional[str] = None,
    on_warning: Optional[Callable[[str], None]] = None,
) -> Tuple[Dict[str, List[Tuple[str, List[str]]]], List[Any]]:
    """Collect validated store/hash/value batches while preserving passthrough items."""
    batch: Dict[str, List[Tuple[str, List[str]]]] = {}
    pass_through: List[Any] = []

    for item in items:
        pass_through.append(item)

        raw_hash = override_hash or get_field(item, "hash")
        raw_store = override_store or get_field(item, "store")
        if not raw_hash or not raw_store:
            if on_warning is not None:
                on_warning("Item missing hash/store; skipping")
            continue

        normalized = normalize_hash(raw_hash)
        if not normalized:
            if on_warning is not None:
                on_warning("Item has invalid hash; skipping")
            continue

        store_text = str(raw_store).strip()
        if not store_text:
            if on_warning is not None:
                on_warning("Item has empty store; skipping")
            continue

        try:
            is_available = bool(store_registry.is_available(store_text))
        except Exception:
            is_available = False
        if not is_available:
            if on_warning is not None:
                on_warning(f"Store '{store_text}' not configured; skipping")
            continue

        values = [str(value).strip() for value in (value_resolver(item) or []) if str(value).strip()]
        if not values:
            continue

        batch.setdefault(store_text, []).append((normalized, values))

    return batch, pass_through


def parse_hash_query(query: Optional[str]) -> List[str]:
    """Parse a unified query string for `hash:` into normalized SHA256 hashes.

    Supported examples:
    - hash:<h1>
    - hash:<h1>,<h2>,<h3>
    - Hash: <h1> <h2> <h3>
    - hash:{<h1>, <h2>}

    Returns:
            List of unique normalized 64-hex SHA256 hashes.
    """
    import re

    q = str(query or "").strip()
    if not q:
        return []

    m = re.match(r"^hash(?:es)?\s*:\s*(.+)$", q, flags=re.IGNORECASE)
    if not m:
        return []

    rest = (m.group(1) or "").strip()
    if rest.startswith("{") and rest.endswith("}"):
        rest = rest[1:-1].strip()
    if rest.startswith("[") and rest.endswith("]"):
        rest = rest[1:-1].strip()

    raw_parts = [p.strip() for p in re.split(r"[\s,]+", rest) if p.strip()]
    out: List[str] = []
    for part in raw_parts:
        h = normalize_hash(part)
        if not h:
            continue
        if h not in out:
            out.append(h)
    return out


def parse_single_hash_query(query: Optional[str]) -> Optional[str]:
    """Parse `hash:` query and require exactly one hash."""
    hashes = parse_hash_query(query)
    if len(hashes) != 1:
        return None
    return hashes[0]


def require_hash_query(
    query: Optional[str],
    error_message: str,
    *,
    log_file: Any = None,
) -> Tuple[List[str], bool]:
    """Parse a multi-hash query and log a caller-provided error on invalid input."""
    hashes = parse_hash_query(query)
    if query and not hashes:
        kwargs = {"file": log_file} if log_file is not None else {}
        log(error_message, **kwargs)
        return [], False
    return hashes, True


def require_single_hash_query(
    query: Optional[str],
    error_message: str,
    *,
    log_file: Any = None,
) -> Tuple[Optional[str], bool]:
    """Parse a single-hash query and log a caller-provided error on invalid input."""
    query_hash = parse_single_hash_query(query)
    if query and not query_hash:
        kwargs = {"file": log_file} if log_file is not None else {}
        log(error_message, **kwargs)
        return None, False
    return query_hash, True


def get_hash_for_operation(
    override_hash: Optional[str],
    result: Any,
    field_name: str = "hash"
) -> Optional[str]:
    """Get normalized hash from override or result object, consolidating common pattern.

    Eliminates repeated pattern: normalize_hash(override) if override else normalize_hash(get_field(result, ...))

    Args:
            override_hash: Hash passed as command argument (takes precedence)
            result: Object containing hash field (fallback)
            field_name: Name of hash field in result object (default: "hash")

    Returns:
            Normalized hash string, or None if neither override nor result provides valid hash
    """
    if override_hash:
        return normalize_hash(override_hash)
    hash_value = (
        get_field(result,
                  field_name) or getattr(result,
                                         field_name,
                                         None) or getattr(result,
                                                          "hash",
                                                          None)
    )
    return normalize_hash(hash_value)


def fetch_hydrus_metadata(
    config: Any,
    hash_hex: str,
    *,
    store_name: Optional[str] = None,
    hydrus_client: Any = None,
    **kwargs,
) -> tuple[Optional[Dict[str,
                         Any]],
           Optional[int]]:
    """Fetch metadata from Hydrus for a given hash, consolidating common fetch pattern.

    Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.

    Args:
            config: Configuration object used to resolve the Hydrus plugin/store
            hash_hex: File hash to fetch metadata for
            store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
            hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
            **kwargs: Additional arguments to pass to client.fetch_file_metadata()
                    Common: include_service_keys_to_tags, include_notes, include_file_url, include_duration, etc.

    Returns:
            Tuple of (metadata_dict, error_code)
            - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable
            - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute())
    """
    client = hydrus_client
    hydrus_provider = None
    try:
        from PluginCore.registry import get_plugin

        hydrus_provider = get_plugin("hydrusnetwork", config)
    except Exception:
        hydrus_provider = None

    if client is None:
        if hydrus_provider is not None:
            try:
                client = hydrus_provider.get_client(
                    store_name=store_name if store_name else None,
                    allow_default=not bool(store_name),
                )
            except Exception as exc:
                if store_name:
                    log(f"Hydrus client unavailable for store '{store_name}': {exc}")
                else:
                    log(f"Hydrus client unavailable: {exc}")
                client = None
        if client is None and store_name:
            log(f"Hydrus client unavailable for store '{store_name}'")
            return None, 1
        if client is None and hydrus_provider is None:
            log("Hydrus provider unavailable")
            return None, 1

    if hydrus_provider is not None:
        try:
            metadata = hydrus_provider.fetch_metadata(
                hash_hex,
                store_name=store_name if store_name else None,
                **kwargs,
            )
        except Exception as exc:
            log(f"Hydrus metadata fetch failed: {exc}")
            return None, 1
        if isinstance(metadata, dict):
            return metadata, 0
        if client is None:
            if store_name:
                log(f"Hydrus client unavailable for store '{store_name}'")
            else:
                log("Hydrus metadata unavailable")
            return None, 1

    try:
        payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs)
    except Exception as exc:
        log(f"Hydrus metadata fetch failed: {exc}")
        return None, 1

    items = payload.get("metadata") if isinstance(payload, dict) else None
    meta = items[0] if (
        isinstance(items,
                   list) and items and isinstance(items[0],
                                                  dict)
    ) else None

    return meta, 0


def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any:
    """Extract a field from either a dict or object with fallback default.

    Handles both dict.get(field) and getattr(obj, field) access patterns.
    Also handles lists by accessing the first element.
    For PipeObjects, checks the extra field as well.
    Used throughout cmdlet to uniformly access fields from mixed types.

    Args:
            obj: Dict, object, or list to extract from
            field: Field name to retrieve
            default: Value to return if field not found (default: None)

    Returns:
            Field value if found, otherwise the default value

    Examples:
            get_field(result, "hash")          # From dict or object
            get_field(result, "table", "unknown")  # With default
    """
    return _item_accessor_get_field(obj, field, default)


def should_show_help(args: Sequence[str]) -> bool:
    """Check if help flag was passed in arguments.

    Consolidates repeated pattern of checking for help flags across cmdlet.

    Args:
            args: Command arguments to check

    Returns:
            True if any help flag is present (-?, /?, --help, -h, help, --cmdlet)

    Examples:
            if should_show_help(args):
                    log(json.dumps(CMDLET, ensure_ascii=False, indent=2))
                    return 0
    """
    try:
        return any(
            str(a).lower() in {"-?",
                               "/?",
                               "--help",
                               "-h",
                               "help",
                               "--cmdlet"} for a in args
        )
    except Exception:
        return False


def looks_like_hash(candidate: Optional[str]) -> bool:
    """Check if a string looks like a SHA256 hash (64 hex chars).

    Args:
            candidate: String to test

    Returns:
            True if the string is 64 lowercase hex characters
    """
    if not isinstance(candidate, str):
        return False
    text = candidate.strip().lower()
    return len(text) == 64 and all(ch in "0123456789abcdef" for ch in text)


def pipeline_item_local_path(item: Any) -> Optional[str]:
    """Extract local file path from a pipeline item.

    Supports both dataclass objects with .path attribute and dicts.
    Returns None for HTTP/HTTPS url.

    Args:
            item: Pipeline item (PipelineItem dataclass, dict, or other)

    Returns:
            Local file path string, or None if item is not a local file
    """
    path_value: Optional[str] = None
    if hasattr(item, "path"):
        path_value = getattr(item, "path", None)
    elif isinstance(item, dict):
        raw = item.get("path") or item.get("url")
        path_value = str(raw) if raw is not None else None
    if not isinstance(path_value, str):
        return None
    text = path_value.strip()
    if not text:
        return None
    if text.lower().startswith(("http://", "https://")):
        return None
    return text


def collect_relationship_labels(
    payload: Any,
    label_stack: List[str] | None = None,
    mapping: Dict[str,
                  str] | None = None
) -> Dict[str,
          str]:
    """Recursively extract hash-to-label mappings from nested relationship data.

    Walks through nested dicts/lists looking for sha256-like strings (64 hex chars)
    and builds a mapping from hash to its path in the structure.

    Example:
            data = {
                    "duplicates": [
                            "abc123...",  # Will be mapped to "duplicates"
                            {"type": "related", "items": ["def456..."]}  # Will be mapped to "duplicates / type / items"
                    ]
            }
            result = collect_relationship_labels(data)
            # result = {"abc123...": "duplicates", "def456...": "duplicates / type / items"}

    Args:
            payload: Nested data structure (dict, list, string, etc.)
            label_stack: Internal use - tracks path during recursion
            mapping: Internal use - accumulates hash->label mappings

    Returns:
            Dict mapping hash strings to their path labels
    """
    if label_stack is None:
        label_stack = []
    if mapping is None:
        mapping = {}

    if isinstance(payload, dict):
        for key, value in payload.items():
            next_stack = label_stack
            if isinstance(key, str) and key:
                formatted = key.replace("_", " ").strip()
                next_stack = label_stack + [formatted]
            collect_relationship_labels(value, next_stack, mapping)
    elif isinstance(payload, (list, tuple, set)):
        for value in payload:
            collect_relationship_labels(value, label_stack, mapping)
    elif isinstance(payload, str) and looks_like_hash(payload):
        hash_value = payload.lower()
        if label_stack:
            label = " / ".join(item for item in label_stack if item)
        else:
            label = "related"
        mapping.setdefault(hash_value, label)

    return mapping


def parse_tag_arguments(arguments: Sequence[str]) -> List[str]:
    """Parse tag arguments from command line tokens.

    - Supports comma-separated tags.
    - Supports pipe namespace shorthand: "artist:A|B|C" -> artist:A, artist:B, artist:C.

    Args:
            arguments: Sequence of argument strings

    Returns:
            List of normalized tag strings (empty strings filtered out)
    """

    def _split_top_level_commas(text: str) -> List[str]:
        segments: List[str] = []
        current: List[str] = []
        paren_depth = 0
        angle_depth = 0
        quote: Optional[str] = None
        escape = False

        for ch in text:
            if escape:
                current.append(ch)
                escape = False
                continue
            if ch == "\\":
                current.append(ch)
                escape = True
                continue
            if quote:
                current.append(ch)
                if ch == quote:
                    quote = None
                continue
            if ch in {"'", '"'}:
                current.append(ch)
                quote = ch
                continue
            if ch == "(":
                paren_depth += 1
                current.append(ch)
                continue
            if ch == ")":
                paren_depth = max(0, paren_depth - 1)
                current.append(ch)
                continue
            if ch == "<":
                angle_depth += 1
                current.append(ch)
                continue
            if ch == ">":
                angle_depth = max(0, angle_depth - 1)
                current.append(ch)
                continue
            if ch == "," and paren_depth == 0 and angle_depth == 0:
                segments.append("".join(current).strip())
                current = []
                continue
            current.append(ch)

        tail = "".join(current).strip()
        if tail or segments:
            segments.append(tail)
        return segments

    def _expand_pipe_namespace(text: str) -> List[str]:
        parts = text.split("|")
        expanded: List[str] = []
        last_ns: Optional[str] = None
        for part in parts:
            segment = part.strip()
            if not segment:
                continue
            if ":" in segment:
                ns, val = segment.split(":", 1)
                ns = ns.strip()
                val = val.strip()
                last_ns = ns or last_ns
                if last_ns and val:
                    expanded.append(f"{last_ns}:{val}")
                elif ns or val:
                    expanded.append(f"{ns}:{val}".strip(":"))
            else:
                if last_ns:
                    expanded.append(f"{last_ns}:{segment}")
                else:
                    expanded.append(segment)
        return expanded

    tags: List[str] = []
    for argument in arguments:
        for token in _split_top_level_commas(str(argument)):
            text = token.strip()
            if not text:
                continue
            # Expand namespace shorthand with pipes
            pipe_expanded = _expand_pipe_namespace(text)
            for entry in pipe_expanded:
                candidate = entry.strip()
                if not candidate:
                    continue
                if ":" in candidate:
                    ns, val = candidate.split(":", 1)
                    ns = ns.strip()
                    val = val.strip()
                    candidate = f"{ns}:{val}" if ns or val else ""
                if candidate:
                    tags.append(candidate)
    return tags


_TAG_VALUE_TEMPLATE_RE = re.compile(r"#\(([^)]+)\)")
_TAG_VALUE_FUNCTION_RE = re.compile(r"<([a-zA-Z_][a-zA-Z0-9_-]*)\((.*?)\)>")


def _normalize_tag_value_template_name(value: Any) -> str:
    text = str(value or "").strip().lower()
    if not text:
        return ""
    try:
        text = re.sub(r"\s+", " ", text).strip()
    except Exception:
        text = " ".join(text.split())
    return text


def _tag_value_template_keys(value: Any) -> list[str]:
    normalized = _normalize_tag_value_template_name(value)
    if not normalized:
        return []

    keys = [normalized]

    trimmed_hash = re.sub(r"\s*#+\s*$", "", normalized).strip()
    if trimmed_hash and trimmed_hash not in keys:
        keys.append(trimmed_hash)

    return keys


def _add_tag_values_to_lookup(lookup: Dict[str, List[str]], tag_text: Any) -> None:
    text = str(tag_text or "").strip()
    if not text or ":" not in text:
        return
    if _TAG_VALUE_TEMPLATE_RE.search(text) or _TAG_VALUE_FUNCTION_RE.search(text):
        return

    namespace, value = text.split(":", 1)
    value_text = str(value or "").strip()
    if not value_text:
        return

    for key in _tag_value_template_keys(namespace):
        values = lookup.setdefault(key, [])
        if value_text not in values:
            values.append(value_text)


def build_tag_value_lookup(
    tags: Optional[Iterable[Any]],
    *,
    result: Any = None,
) -> Dict[str, List[str]]:
    """Build a placeholder lookup from existing tags and lightweight result fields.

    Placeholder lookups use ``#(namespace)`` syntax. Namespace matching is
    case-insensitive and trims repeated whitespace. A trailing ``#`` in the
    placeholder is ignored so inputs like ``#(track #)`` can resolve ``track:9``.
    """

    lookup: Dict[str, List[str]] = {}
    for tag in tags or []:
        _add_tag_values_to_lookup(lookup, tag)

    title_text = extract_title_from_result(result)
    if title_text:
        _add_tag_values_to_lookup(lookup, f"title:{title_text}")

    return lookup


def _split_tag_value_function_args(value: Any) -> list[str]:
    text = str(value or "")
    args: list[str] = []
    current: list[str] = []
    depth = 0
    quote: Optional[str] = None
    escape = False

    for ch in text:
        if escape:
            current.append(ch)
            escape = False
            continue
        if ch == "\\":
            current.append(ch)
            escape = True
            continue
        if quote:
            current.append(ch)
            if ch == quote:
                quote = None
            continue
        if ch in {"'", '"'}:
            current.append(ch)
            quote = ch
            continue
        if ch in {"(", "[", "{"}:
            depth += 1
            current.append(ch)
            continue
        if ch in {")", "]", "}"}:
            depth = max(0, depth - 1)
            current.append(ch)
            continue
        if ch == "," and depth == 0:
            args.append("".join(current).strip())
            current = []
            continue
        current.append(ch)

    tail = "".join(current).strip()
    if tail or args:
        args.append(tail)
    return args


def _strip_tag_value_function_arg(value: Any) -> str:
    text = str(value or "").strip()
    if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
        return text[1:-1]
    return text


def _padding_width_from_spec(value: Any) -> Optional[int]:
    spec = _strip_tag_value_function_arg(value)
    if not spec:
        return None
    if re.fullmatch(r"0+", spec):
        return len(spec)
    if spec.isdigit():
        try:
            width = int(spec)
        except Exception:
            return None
        return width if width > 0 else None
    return None


def _replace_tag_value_placeholders(
    value: Any,
    lookup: Dict[str, List[str]],
    *,
    preserve_unresolved: bool,
) -> tuple[str, bool]:
    text = str(value or "")
    unresolved = False

    def _replace(match: re.Match[str]) -> str:
        nonlocal unresolved
        keys = _tag_value_template_keys(match.group(1) or "")
        values: List[str] = []
        for key in keys:
            for candidate in lookup.get(key, []):
                if candidate not in values:
                    values.append(candidate)
        if not values:
            unresolved = True
            return match.group(0) if preserve_unresolved else ""
        return ", ".join(values)

    return _TAG_VALUE_TEMPLATE_RE.sub(_replace, text), unresolved


def _coerce_tag_value_integer(value: Any) -> Optional[int]:
    text = _strip_tag_value_function_arg(value)
    if not text:
        return None
    if not re.fullmatch(r"[+-]?\d+", text):
        return None
    try:
        return int(text)
    except Exception:
        return None


def _apply_tag_value_function(
    name: str,
    args: Sequence[str],
    *,
    lookup: Dict[str, List[str]],
) -> Optional[str]:
    func = str(name or "").strip().lower()

    resolved_values: list[str] = []
    unresolved_flags: list[bool] = []
    for arg in args:
        rendered, unresolved = _replace_tag_value_placeholders(
            arg,
            lookup,
            preserve_unresolved=True,
        )
        resolved_values.append(_strip_tag_value_function_arg(rendered))
        unresolved_flags.append(unresolved)

    if func in {"padding", "pad", "zfill"}:
        if len(resolved_values) != 2 or any(unresolved_flags):
            return None
        width = _padding_width_from_spec(resolved_values[0])
        if width is None:
            return None
        return str(resolved_values[1]).zfill(width)

    if func == "default":
        if len(resolved_values) != 2:
            return None
        primary = resolved_values[0]
        fallback = resolved_values[1]
        if not unresolved_flags[0] and str(primary).strip():
            return str(primary)
        if unresolved_flags[1]:
            return None
        return str(fallback)

    if func == "replace":
        if len(resolved_values) != 3 or any(unresolved_flags):
            return None
        return str(resolved_values[0]).replace(
            str(resolved_values[1]),
            str(resolved_values[2]),
        )

    if func in {"increment", "inc", "add"}:
        if len(resolved_values) not in {1, 2}:
            return None
        if unresolved_flags[0]:
            return None
        base_value = _coerce_tag_value_integer(resolved_values[0])
        if base_value is None:
            return None
        step_value = 1
        if len(resolved_values) == 2:
            if unresolved_flags[1]:
                return None
            parsed_step = _coerce_tag_value_integer(resolved_values[1])
            if parsed_step is None:
                return None
            step_value = parsed_step
        return str(base_value + step_value)

    return None


def _render_tag_value_function_templates(
    value: Any,
    *,
    lookup: Dict[str, List[str]],
) -> tuple[str, bool]:
    text = str(value or "")
    unresolved = False

    def _replace(match: re.Match[str]) -> str:
        nonlocal unresolved
        func_name = match.group(1) or ""
        func_args = _split_tag_value_function_args(match.group(2) or "")
        rendered = _apply_tag_value_function(
            func_name,
            func_args,
            lookup=lookup,
        )
        if rendered is None:
            unresolved = True
            return match.group(0)
        return rendered

    previous = None
    rendered = text
    while previous != rendered and _TAG_VALUE_FUNCTION_RE.search(rendered):
        previous = rendered
        rendered = _TAG_VALUE_FUNCTION_RE.sub(_replace, rendered)
        if unresolved:
            break
    return rendered, unresolved


def render_tag_value_templates(
    tags: Sequence[Any],
    *,
    existing_tags: Optional[Iterable[Any]] = None,
    result: Any = None,
) -> tuple[list[str], list[str]]:
    """Resolve ``#(namespace)`` placeholders and ``<transform(...)>`` functions.

    Returns ``(resolved_tags, unresolved_templates)``. Tags whose placeholders
    cannot be fully resolved are omitted from ``resolved_tags`` and returned in
    ``unresolved_templates`` so callers can warn or summarize skipped items.

    Currently supported transforms:
    - ``<padding(00,#(episode))>`` or ``<pad(2,#(episode))>`` for zero-padding
    - ``<default(#(season),0)>`` to fall back when a placeholder is missing
    - ``<replace(#(title),old,new)>`` for simple substring replacement
    - ``<increment(#(episode),1)>`` for integer arithmetic
    """

    entries: list[dict[str, Any]] = []
    lookup = build_tag_value_lookup(existing_tags, result=result)

    for raw_tag in tags or []:
        text = str(raw_tag or "").strip()
        if not text:
            continue
        has_template = bool(
            _TAG_VALUE_TEMPLATE_RE.search(text)
            or _TAG_VALUE_FUNCTION_RE.search(text)
        )
        entry = {
            "raw": text,
            "resolved": None,
            "has_template": has_template,
        }
        if not has_template:
            entry["resolved"] = text
            _add_tag_values_to_lookup(lookup, text)
        entries.append(entry)

    progress = True
    while progress:
        progress = False
        for entry in entries:
            if entry["resolved"] is not None or not entry["has_template"]:
                continue

            rendered, unresolved = _replace_tag_value_placeholders(
                entry["raw"],
                lookup,
                preserve_unresolved=bool(_TAG_VALUE_FUNCTION_RE.search(str(entry["raw"]))),
            )

            rendered, function_unresolved = _render_tag_value_function_templates(
                rendered,
                lookup=lookup,
            )
            if function_unresolved:
                continue

            if unresolved and _TAG_VALUE_TEMPLATE_RE.search(rendered):
                continue

            rendered = rendered.strip()
            if not rendered:
                entry["resolved"] = ""
                progress = True
                continue

            entry["resolved"] = rendered
            _add_tag_values_to_lookup(lookup, rendered)
            progress = True

    resolved_tags = merge_sequences(
        [entry["resolved"] for entry in entries if isinstance(entry.get("resolved"), str) and entry.get("resolved")],
        case_sensitive=True,
    )
    unresolved_templates = [
        str(entry["raw"])
        for entry in entries
        if entry["has_template"] and not entry.get("resolved")
    ]
    return resolved_tags, unresolved_templates


def fmt_bytes(n: Optional[int]) -> str:
    """Format bytes as human-readable with 1 decimal place (MB/GB).

    Args:
            n: Number of bytes, or None

    Returns:
            Formatted string like "1.5 MB" or "2.0 GB", or "unknown"
    """
    if n is None or n < 0:
        return "unknown"
    gb = n / (1024.0 * 1024.0 * 1024.0)
    if gb >= 1.0:
        return f"{gb:.1f} GB"
    mb = n / (1024.0 * 1024.0)
    return f"{mb:.1f} MB"


def _normalize_tag_group_entry(value: Any) -> Optional[str]:
    """Internal: Normalize a single tag group entry."""
    if not isinstance(value, str):
        value = str(value)
    text = value.strip()
    return text or None


def _load_tag_groups() -> Dict[str, List[str]]:
    """Load tag group definitions from JSON file with caching."""
    global _TAG_GROUPS_CACHE, _TAG_GROUPS_MTIME, TAG_GROUPS_PATH

    # Auto-discover adjective.json if not set
    if TAG_GROUPS_PATH is None:
        # Try to find adjective.json in the script directory or helper subdirectory
        try:
            script_dir = Path(__file__).parent.parent

            # Check root directory
            candidate = script_dir / "adjective.json"
            if candidate.exists():
                TAG_GROUPS_PATH = candidate
            else:
                # Check helper directory
                candidate = script_dir / "helper" / "adjective.json"
                if candidate.exists():
                    TAG_GROUPS_PATH = candidate
        except Exception:
            pass

    if TAG_GROUPS_PATH is None:
        return {}

    path = TAG_GROUPS_PATH
    try:
        stat_result = path.stat()
    except FileNotFoundError:
        _TAG_GROUPS_CACHE = {}
        _TAG_GROUPS_MTIME = None
        return {}
    except OSError as exc:
        log(f"Failed to read tag groups: {exc}", file=sys.stderr)
        _TAG_GROUPS_CACHE = {}
        _TAG_GROUPS_MTIME = None
        return {}

    mtime = stat_result.st_mtime
    if _TAG_GROUPS_CACHE is not None and _TAG_GROUPS_MTIME == mtime:
        return _TAG_GROUPS_CACHE

    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as exc:
        log(f"Invalid tag group JSON ({path}): {exc}", file=sys.stderr)
        _TAG_GROUPS_CACHE = {}
        _TAG_GROUPS_MTIME = mtime
        return {}

    groups: Dict[str,
                 List[str]] = {}
    if isinstance(payload, dict):
        for key, value in payload.items():
            if not isinstance(key, str):
                continue
            name = key.strip().lower()
            if not name:
                continue
            members: List[str] = []
            if isinstance(value, list):
                for entry in value:
                    normalized = _normalize_tag_group_entry(entry)
                    if normalized:
                        members.append(normalized)
            elif isinstance(value, str):
                normalized = _normalize_tag_group_entry(value)
                if normalized:
                    members.extend(
                        token.strip() for token in normalized.split(",")
                        if token.strip()
                    )
            if members:
                groups[name] = members

    _TAG_GROUPS_CACHE = groups
    _TAG_GROUPS_MTIME = mtime
    return groups


def expand_tag_groups(raw_tags: Iterable[str]) -> List[str]:
    """Expand tag group references (e.g., {my_group}) into member tags.

    Tag groups are defined in JSON and can be nested. Groups are referenced
    with curly braces: {group_name}.

    Args:
            raw_tags: Sequence of tag strings, some may reference groups like "{group_name}"

    Returns:
            List of expanded tags with group references replaced
    """
    groups = _load_tag_groups()
    if not groups:
        return [tag for tag in raw_tags if isinstance(tag, str) and tag.strip()]

    def _expand(tokens: Iterable[str], seen: Set[str]) -> List[str]:
        result: List[str] = []
        for token in tokens:
            if not isinstance(token, str):
                continue
            candidate = token.strip()
            if not candidate:
                continue
            if candidate.startswith("{") and candidate.endswith("}") and len(candidate
                                                                             ) > 2:
                name = candidate[1:-1].strip().lower()
                if not name:
                    continue
                if name in seen:
                    log(
                        f"Tag group recursion detected for {{{name}}}; skipping",
                        file=sys.stderr
                    )
                    continue
                members = groups.get(name)
                if not members:
                    log(f"Unknown tag group {{{name}}}", file=sys.stderr)
                    result.append(candidate)
                    continue
                result.extend(_expand(members,
                                      seen | {name}))
            else:
                result.append(candidate)
        return result

    return _expand(raw_tags, set())


def first_title_tag(source: Optional[Iterable[str]]) -> Optional[str]:
    """Find the first tag starting with "title:" in a collection.

    Args:
            source: Iterable of tag strings

    Returns:
            First title: tag found, or None
    """
    if not source:
        return None
    for item in source:
        if not isinstance(item, str):
            continue
        candidate = item.strip()
        if candidate and candidate.lower().startswith("title:"):
            return candidate
    return None


def apply_preferred_title(tags: List[str], preferred: Optional[str]) -> List[str]:
    """Replace any title: tags with a preferred title tag.

    Args:
            tags: List of tags (may contain multiple "title:" entries)
            preferred: Preferred title tag to use (full "title: ..." format)

    Returns:
            List with old title tags removed and preferred title added (at most once)
    """
    if not preferred:
        return tags
    preferred_clean = preferred.strip()
    if not preferred_clean:
        return tags
    preferred_lower = preferred_clean.lower()
    filtered: List[str] = []
    has_preferred = False
    for tag in tags:
        candidate = tag.strip()
        if not candidate:
            continue
        if candidate.lower().startswith("title:"):
            if candidate.lower() == preferred_lower:
                if not has_preferred:
                    filtered.append(candidate)
                    has_preferred = True
            continue
        filtered.append(candidate)
    if not has_preferred:
        filtered.append(preferred_clean)
    return filtered


# ============================================================================
# PIPEOBJECT UTILITIES (for chainable cmdlet and multi-action pipelines)
# ============================================================================


def create_pipe_object_result(
    source: str,
    identifier: str,
    file_path: str,
    cmdlet_name: str,
    title: Optional[str] = None,
    hash_value: Optional[str] = None,
    is_temp: bool = False,
    parent_hash: Optional[str] = None,
    tag: Optional[List[str]] = None,
    **extra: Any,
) -> Dict[str,
          Any]:
    """Create a PipeObject-compatible result dict for pipeline chaining.

    This is a helper to emit results in the standard format that downstream
    cmdlet can process (filter, tag, cleanup, etc.).

    Args:
            source: Source system (e.g., 'local', 'hydrus', 'download')
            identifier: Unique ID from source
            file_path: Path to the file
            cmdlet_name: Name of the cmdlet that created this (e.g., 'download-data', 'screen-shot')
            title: Human-readable title
            hash_value: SHA-256 hash of file (for integrity)
            is_temp: If True, this is a temporary/intermediate artifact
            parent_hash: Hash of the parent file in the chain (for provenance)
            tag: List of tag values to apply
            **extra: Additional fields

    Returns:
            Dict with all PipeObject fields for emission
    """
    store_override = extra.pop("store", None)

    result = build_file_result_payload(
        title=title,
        path=file_path,
        hash_value=hash_value,
        store=store_override if store_override is not None else source,
        tag=tag,
        source=source,
        id=identifier,
        action=f"cmdlet:{cmdlet_name}",
        **extra,
    )

    if is_temp:
        result["is_temp"] = True
    if parent_hash:
        result["parent_hash"] = parent_hash

    return result


def mark_as_temp(pipe_object: Dict[str, Any]) -> Dict[str, Any]:
    """Mark a PipeObject dict as temporary (intermediate artifact).

    Args:
            pipe_object: Result dict from cmdlet emission

    Returns:
            Modified dict with is_temp=True
    """
    pipe_object["is_temp"] = True
    return pipe_object


def set_parent_hash(pipe_object: Dict[str, Any], parent_hash: str) -> Dict[str, Any]:
    """Set the parent_hash for provenance tracking.

    Args:
            pipe_object: Result dict
            parent_hash: Parent file's hash

    Returns:
            Modified dict with parent_hash set to the hash
    """
    pipe_object["parent_hash"] = parent_hash
    return pipe_object


def get_pipe_object_path(pipe_object: Any) -> Optional[str]:
    """Extract file path from PipeObject, dict, or pipeline-friendly object."""
    if pipe_object is None:
        return None
    for attr in ("path", "target"):
        if hasattr(pipe_object, attr):
            value = getattr(pipe_object, attr)
            if value:
                return value
    if isinstance(pipe_object, dict):
        for key in ("path", "target"):
            value = pipe_object.get(key)
            if value:
                return value
    return None


def _extract_flag_value(args: Sequence[str], *flags: str) -> Optional[str]:
    """Return the value for the first matching flag in args.

    This is intentionally lightweight (no cmdlet spec required) so callers in CLI/pipeline
    can share the same behavior.
    """
    if not args:
        return None
    want = {str(f).strip().lower()
            for f in flags if str(f).strip()}
    if not want:
        return None
    try:
        tokens = [str(a) for a in args]
    except Exception:
        tokens = list(args)  # type: ignore[list-item]
    for i, tok in enumerate(tokens):
        low = str(tok).strip().lower()
        if low in want:
            if i + 1 >= len(tokens):
                return None
            nxt = str(tokens[i + 1])
            # Allow paths like "-"? Treat missing value as None.
            if not nxt.strip():
                return None
            # Don't consume another flag as value.
            if nxt.startswith("-"):
                return None
            return nxt
    return None


def _unique_destination_path(dest: Path) -> Path:
    """Generate a non-colliding destination path by appending " (N)"."""
    try:
        if not dest.exists():
            return dest
    except Exception:
        return dest

    parent = dest.parent
    stem = dest.stem
    suffix = dest.suffix
    for i in range(1, 10_000):
        candidate = parent / f"{stem} ({i}){suffix}"
        try:
            if not candidate.exists():
                return candidate
        except Exception:
            return candidate
    return dest


def _print_live_safe_stderr(message: str) -> None:
    """Print to stderr without breaking Rich Live progress output."""
    try:
        from SYS.rich_display import stderr_console  # type: ignore
    except Exception:
        return

    cm: AbstractContextManager[Any] | None = None
    try:
        from SYS import pipeline as _pipeline_ctx  # type: ignore

        suspend = getattr(_pipeline_ctx, "suspend_live_progress", None)
        candidate = suspend() if callable(suspend) else None
        if isinstance(candidate, AbstractContextManager):
            cm = candidate
        elif candidate is not None and hasattr(candidate, "__enter__") and hasattr(candidate, "__exit__"):
            cm = candidate  # type: ignore[arg-type]
    except Exception:
        cm = None

    if cm is None:
        cm = nullcontext()

    try:
        console = stderr_console()
        print_func = getattr(console, "print", None)
    except Exception:
        return

    if not callable(print_func):
        return

    try:
        with cm:
            print_func(str(message))
    except Exception:
        return


def apply_output_path_from_pipeobjects(
    *,
    cmd_name: str,
    args: Sequence[str],
    emits: Sequence[Any],
) -> List[Any]:
    """If the user supplied `-path`, move emitted temp/PATH files there.

    This enables a dynamic pattern:
    - Any cmdlet can include `SharedArgs.PATH`.
    - If it emits a file-backed PipeObject (`path` exists on disk) and the item is
      a temp/PATH artifact, then `-path <dest>` will save it to that location.

    Rules:
    - Only affects items whose `action` matches the current cmdlet.
    - Only affects items that look like local artifacts (`is_temp` True or `store` == PATH).
    - Updates the emitted object's `path` (and `target` when it points at the same file).
    """
    dest_raw = _extract_flag_value(args, "-path", "--path")
    if not dest_raw:
        return list(emits or [])

    # Guard: users sometimes pass a URL into -path by mistake (e.g. `-path https://...`).
    # Treat that as invalid for filesystem moves and avoid breaking Rich Live output.
    try:
        dest_str = str(dest_raw).strip()
        if "://" in dest_str:
            _print_live_safe_stderr(
                f"Ignoring -path value that looks like a URL: {dest_str}"
            )
            return list(emits or [])
    except Exception:
        pass

    cmd_norm = str(cmd_name or "").replace("_", "-").strip().lower()
    if not cmd_norm:
        return list(emits or [])

    try:
        dest_hint_dir = str(dest_raw).endswith(("/", "\\"))
    except Exception:
        dest_hint_dir = False

    try:
        dest_path = Path(str(dest_raw)).expanduser()
    except Exception:
        return list(emits or [])

    items = list(emits or [])
    # Identify which emitted items are actually file artifacts produced by this cmdlet.
    artifact_indices: List[int] = []
    artifact_paths: List[Path] = []
    for idx, item in enumerate(items):
        action = str(get_field(item, "action", "") or "").strip().lower()
        if not action.startswith("cmdlet:"):
            continue
        action_name = action.split(":", 1)[-1].strip().lower()
        if action_name != cmd_norm:
            continue

        store = str(get_field(item, "store", "") or "").strip().lower()
        is_temp = bool(get_field(item, "is_temp", False))
        if not (is_temp or store == "path"):
            continue

        src_str = get_pipe_object_path(item)
        if not src_str:
            continue
        try:
            src = Path(str(src_str)).expanduser()
        except Exception:
            continue
        try:
            if not src.exists() or not src.is_file():
                continue
        except Exception:
            continue

        artifact_indices.append(idx)
        artifact_paths.append(src)

    if not artifact_indices:
        return items

    # Decide whether the destination is a directory or a single file.
    if len(artifact_indices) > 1:
        # Multiple artifacts: always treat destination as a directory.
        if dest_path.suffix:
            dest_dir = dest_path.parent
        else:
            dest_dir = dest_path
        try:
            dest_dir.mkdir(parents=True, exist_ok=True)
        except Exception as exc:
            _print_live_safe_stderr(
                f"Failed to create destination directory: {dest_dir} ({exc})"
            )
            return items

        for idx, src in zip(artifact_indices, artifact_paths):
            final = dest_dir / src.name
            final = _unique_destination_path(final)
            try:
                if src.resolve() == final.resolve():
                    _apply_saved_path_update(
                        items[idx],
                        old_path=str(src),
                        new_path=str(final)
                    )
                    _print_saved_output_panel(items[idx], final)
                    continue
            except Exception:
                pass
            try:
                shutil.move(str(src), str(final))
            except Exception as exc:
                _print_live_safe_stderr(f"Failed to save output to {final}: {exc}")
                continue
            _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final))
            _print_saved_output_panel(items[idx], final)

        return items

    # Single artifact: destination can be a directory or a concrete file path.
    src = artifact_paths[0]
    idx = artifact_indices[0]
    final: Path
    try:
        if dest_hint_dir or (dest_path.exists() and dest_path.is_dir()):
            final = dest_path / src.name
        else:
            final = dest_path
    except Exception:
        final = dest_path

    try:
        final.parent.mkdir(parents=True, exist_ok=True)
    except Exception as exc:
        _print_live_safe_stderr(
            f"Failed to create destination directory: {final.parent} ({exc})"
        )
        return items

    final = _unique_destination_path(final)
    try:
        if src.resolve() != final.resolve():
            shutil.move(str(src), str(final))
    except Exception as exc:
        _print_live_safe_stderr(f"Failed to save output to {final}: {exc}")
        return items

    _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final))
    _print_saved_output_panel(items[idx], final)
    return items


def _print_saved_output_panel(item: Any, final_path: Path) -> None:
    """When -path is used, print a Rich panel summarizing the saved output.

    Shows: Title, Location, Hash.
    Best-effort: reads existing fields first to avoid recomputing hashes.
    """
    try:
        from rich.panel import Panel  # type: ignore
        from rich.table import Table  # type: ignore
        from SYS.rich_display import stderr_console  # type: ignore
    except Exception:
        return

    # If Rich Live progress is active, pause it while printing so the panel
    # doesn't get overwritten/truncated by Live's cursor control.
    cm: AbstractContextManager[Any] | None = None
    try:
        from SYS import pipeline as _pipeline_ctx  # type: ignore

        suspend = getattr(_pipeline_ctx, "suspend_live_progress", None)
        cm_candidate = suspend() if callable(suspend) else None
        if isinstance(cm_candidate, AbstractContextManager):
            cm = cm_candidate
        elif cm_candidate is not None and hasattr(cm_candidate, "__enter__") and hasattr(cm_candidate, "__exit__"):
            cm = cm_candidate  # type: ignore[arg-type]
    except Exception:
        cm = None

    if cm is None:
        cm = nullcontext()

    try:
        location = str(final_path)
    except Exception:
        location = ""

    title = ""
    try:
        title = str(get_field(item, "title") or get_field(item, "name") or "").strip()
    except Exception:
        title = ""
    if not title:
        try:
            title = str(final_path.stem or final_path.name)
        except Exception:
            title = ""

    file_hash = ""
    try:
        file_hash = str(get_field(item,
                                  "hash") or get_field(item,
                                                       "sha256") or "").strip()
    except Exception:
        file_hash = ""
    if not file_hash:
        try:
            from SYS.utils import sha256_file  # type: ignore

            file_hash = str(sha256_file(final_path) or "").strip()
        except Exception:
            file_hash = ""

    grid = Table.grid(padding=(0, 1))
    grid.add_column(justify="right", style="bold")
    grid.add_column()
    grid.add_row("Title", title or "(unknown)")
    grid.add_row("Location", location or "(unknown)")
    grid.add_row("Hash", file_hash or "(unknown)")

    try:
        console = stderr_console()
        print_func = getattr(console, "print", None)
    except Exception:
        return

    if not callable(print_func):
        return

    try:
        with cm:
            print_func(Panel(grid, title="Saved", expand=False))
    except Exception:
        return


def _apply_saved_path_update(item: Any, *, old_path: str, new_path: str) -> None:
    """Update a PipeObject-like item after its backing file has moved."""
    old_str = str(old_path)
    new_str = str(new_path)
    if isinstance(item, dict):
        try:
            if str(item.get("path") or "") == old_str:
                item["path"] = new_str
        except Exception:
            pass
        try:
            if str(item.get("target") or "") == old_str:
                item["target"] = new_str
        except Exception:
            pass
        try:
            extra = item.get("extra")
            if isinstance(extra, dict):
                if str(extra.get("target") or "") == old_str:
                    extra["target"] = new_str
                if str(extra.get("path") or "") == old_str:
                    extra["path"] = new_str
        except Exception:
            pass
        return

    # models.PipeObject or PipeObject-ish
    try:
        if getattr(item, "path", None) == old_str:
            setattr(item, "path", new_str)
    except Exception:
        pass
    try:
        extra = getattr(item, "extra", None)
        if isinstance(extra, dict):
            if str(extra.get("target") or "") == old_str:
                extra["target"] = new_str
            if str(extra.get("path") or "") == old_str:
                extra["path"] = new_str
    except Exception:
        pass


def get_pipe_object_hash(pipe_object: Any) -> Optional[str]:
    """Extract file hash from PipeObject, dict, or pipeline-friendly object."""
    if pipe_object is None:
        return None
    for attr in ("hash",
                 ):
        if hasattr(pipe_object, attr):
            value = getattr(pipe_object, attr)
            if value:
                return value
    if isinstance(pipe_object, dict):
        for key in ("hash",
                    ):
            value = pipe_object.get(key)
            if value:
                return value
    return None


def normalize_result_input(result: Any) -> List[Dict[str, Any]]:
    """Normalize input result to a list of dicts.

    Handles:
    - None -> []
    - Dict -> [dict]
    - List of dicts -> list as-is
    - PipeObject -> [dict]
    - List of PipeObjects -> list of dicts

    Args:
            result: Result from piped input

    Returns:
            List of result dicts (may be empty)
    """
    if result is None:
        return []

    # Single dict
    if isinstance(result, dict):
        return [result]

    # List - convert each item to dict if needed
    if isinstance(result, list):
        output = []
        for item in result:
            if isinstance(item, dict):
                output.append(item)
            elif hasattr(item, "to_dict"):
                output.append(item.to_dict())
            else:
                # Try as-is
                output.append(item)
        return output

    # PipeObject or other object with to_dict
    if hasattr(result, "to_dict"):
        return [result.to_dict()]

    # Fallback: wrap it
    if isinstance(result, dict):
        return [result]

    return []


def normalize_result_items(
    result: Any,
    *,
    include_falsey_single: bool = False,
) -> List[Any]:
    """Normalize piped input to a raw item list without converting item types."""
    if isinstance(result, list):
        return list(result)
    if result is None:
        return []
    if include_falsey_single or result:
        return [result]
    return []


def value_has_content(value: Any) -> bool:
    """Return True when a value should be treated as present for payload building."""
    if value is None:
        return False
    if isinstance(value, str):
        return bool(value.strip())
    if isinstance(value, (list, tuple, set)):
        return len(value) > 0
    return True


def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]:
    """Filter results by temporary status.

    Args:
            results: List of result dicts or PipeObjects
            include_temp: If True, keep temp files; if False, exclude them

    Returns:
            Filtered list
    """
    if include_temp:
        return results

    filtered = []
    for result in results:
        is_temp = False

        # Check PipeObject
        if hasattr(result, "is_temp"):
            is_temp = result.is_temp
        # Check dict
        elif isinstance(result, dict):
            is_temp = result.get("is_temp", False)

        if not is_temp:
            filtered.append(result)

    return filtered


def merge_sequences(*sources: Optional[Iterable[Any]],
                    case_sensitive: bool = True) -> list[str]:
    """Merge iterable sources while preserving order and removing duplicates."""
    seen: set[str] = set()
    merged: list[str] = []
    for source in sources:
        if not source:
            continue
        if isinstance(source, str) or not isinstance(source, IterableABC):
            iterable = [source]
        else:
            iterable = source
        for value in iterable:
            if value is None:
                continue
            text = str(value).strip()
            if not text:
                continue
            key = text if case_sensitive else text.lower()
            if key in seen:
                continue
            seen.add(key)
            merged.append(text)
    return merged


def collapse_namespace_tags(
    tags: Optional[Iterable[Any]],
    namespace: str,
    prefer: str = "last"
) -> list[str]:
    """Reduce tags so only one entry for a given namespace remains.

    Keeps either the first or last occurrence (default last) while preserving overall order
    for non-matching tags. Useful for ensuring a single title: tag.
    """
    if not tags:
        return []
    ns = str(namespace or "").strip().lower()
    if not ns:
        return list(tags) if isinstance(tags, list) else list(tags)

    prefer_last = str(prefer or "last").lower() != "first"
    ns_prefix = ns + ":"

    items = list(tags)
    if prefer_last:
        kept: list[str] = []
        seen_ns = False
        for tag in reversed(items):
            text = str(tag)
            if text.lower().startswith(ns_prefix):
                if seen_ns:
                    continue
                seen_ns = True
            kept.append(text)
        kept.reverse()
        return kept
    else:
        kept_ns = False
        result: list[str] = []
        for tag in items:
            text = str(tag)
            if text.lower().startswith(ns_prefix):
                if kept_ns:
                    continue
                kept_ns = True
            result.append(text)
        return result
def extract_tag_from_result(result: Any) -> list[str]:
    """Extract all tags from a result dict or PipeObject.

    Handles mixed types (lists, sets, strings) and various field names.
    """
    tag: list[str] = []

    def _extend(candidate: Any) -> None:
        if not candidate:
            return
        if isinstance(candidate, (list, set, tuple)):
            tag.extend(str(t) for t in candidate if t is not None)
        elif isinstance(candidate, str):
            tag.append(candidate)

    if isinstance(result, models.PipeObject):
        tag.extend(result.tag or [])
        if isinstance(result.extra, dict):
            _extend(result.extra.get("tag"))
        if isinstance(result.metadata, dict):
            _extend(result.metadata.get("tag"))
            _extend(result.metadata.get("tags"))
    elif hasattr(result, "tag"):
        # Handle objects with tag attribute (e.g. SearchResult)
        _extend(getattr(result, "tag"))

    if isinstance(result, dict):
        _extend(result.get("tag"))
        _extend(result.get("tags"))
        
        extra = result.get("extra")
        if isinstance(extra, dict):
            _extend(extra.get("tag"))
            _extend(extra.get("tags"))
            
        fm = result.get("full_metadata") or result.get("metadata")
        if isinstance(fm, dict):
            _extend(fm.get("tag"))
            _extend(fm.get("tags"))

    return merge_sequences(tag, case_sensitive=True)


def extract_title_from_result(result: Any) -> Optional[str]:
    """Extract the title from a result dict or PipeObject."""
    if isinstance(result, models.PipeObject):
        return result.title
    elif hasattr(result, "title"):
        return getattr(result, "title")
    elif isinstance(result, dict):
        return result.get("title")
    return None


def extract_url_from_result(result: Any) -> list[str]:
    """Extract all unique URLs from a result dict or PipeObject.

    Handles mixed types (lists, strings) and various field names (url, source_url, webpage_url).
    Centralizes extraction logic for cmdlets like download-file, add-file, get-url.
    """
    url: list[str] = []

    def _extend(candidate: Any) -> None:
        if not candidate:
            return
        if isinstance(candidate, list):
            url.extend(candidate)
        elif isinstance(candidate, str):
            url.append(candidate)

    # Priority 1: PipeObject (structured data)
    if isinstance(result, models.PipeObject):
        _extend(result.url)
        _extend(result.source_url)
        # Also check extra and metadata for legacy or rich captures
        if isinstance(result.extra, dict):
            _extend(result.extra.get("url"))
            _extend(result.extra.get("source_url"))
        if isinstance(result.metadata, dict):
            _extend(result.metadata.get("url"))
            _extend(result.metadata.get("source_url"))
            _extend(result.metadata.get("webpage_url"))
        if isinstance(getattr(result, "full_metadata", None), dict):
            fm = getattr(result, "full_metadata", None)
            if isinstance(fm, dict):
                _extend(fm.get("url"))
                _extend(fm.get("source_url"))
                _extend(fm.get("webpage_url"))

    # Priority 2: Generic objects with .url or .source_url attribute
    elif hasattr(result, "url") or hasattr(result, "source_url"):
        _extend(getattr(result, "url", None))
        _extend(getattr(result, "source_url", None))

    # Priority 3: Dictionary
    if isinstance(result, dict):
        _extend(result.get("url"))
        _extend(result.get("source_url"))
        _extend(result.get("webpage_url"))
        
        extra = result.get("extra")
        if isinstance(extra, dict):
            _extend(extra.get("url"))
            
        fm = result.get("full_metadata") or result.get("metadata")
        if isinstance(fm, dict):
            _extend(fm.get("url"))
            _extend(fm.get("source_url"))
            _extend(fm.get("webpage_url"))

    from SYS.metadata import normalize_urls
    return normalize_urls(url)


def merge_urls(existing: Any, incoming: Sequence[Any]) -> list[str]:
    """Merge URL values into a normalized, de-duplicated list."""
    from SYS.metadata import normalize_urls

    merged: list[str] = []
    for value in normalize_urls(existing):
        if value not in merged:
            merged.append(value)
    for value in normalize_urls(list(incoming or [])):
        if value not in merged:
            merged.append(value)
    return merged


def remove_urls(existing: Any, remove: Sequence[Any]) -> list[str]:
    """Remove URL values from an existing URL field and return survivors."""
    from SYS.metadata import normalize_urls

    current = normalize_urls(existing)
    remove_set = {value for value in normalize_urls(list(remove or [])) if value}
    if not remove_set:
        return current
    return [value for value in current if value not in remove_set]


def set_item_urls(item: Any, urls: Sequence[Any]) -> None:
    """Persist normalized URL values back onto a dict/object result item."""
    normalized = merge_urls([], list(urls or []))
    payload: Any = normalized[0] if len(normalized) == 1 else list(normalized)

    try:
        if isinstance(item, dict):
            item["url"] = payload
            return
        if hasattr(item, "url"):
            setattr(item, "url", payload)
    except Exception:
        return


def extract_relationships(result: Any) -> Optional[Dict[str, Any]]:
    if isinstance(result, models.PipeObject):
        relationships = result.get_relationships()
        return relationships or None
    if isinstance(result, dict):
        relationships = result.get("relationships")
        if isinstance(relationships, dict) and relationships:
            return relationships
    return None


def extract_duration(result: Any) -> Optional[float]:
    duration = None
    if isinstance(result, models.PipeObject):
        duration = result.duration
    elif isinstance(result, dict):
        duration = result.get("duration")
        if duration is None:
            metadata = result.get("metadata")
            if isinstance(metadata, dict):
                duration = metadata.get("duration")
    if duration is None:
        return None
    try:
        return float(duration)
    except (TypeError, ValueError):
        return None


def coerce_to_pipe_object(
    value: Any,
    default_path: Optional[str] = None
) -> models.PipeObject:
    """Normalize any incoming result to a PipeObject for single-source-of-truth state.

    Uses hash+store canonical pattern.
    """
    # Debug: Print ResultItem details if coming from search_file.py
    try:
        from SYS.logger import is_debug_enabled, debug

        if (is_debug_enabled() and hasattr(value,
                                           "__class__")
                and value.__class__.__name__ == "ResultItem"):
            debug("[ResultItem -> PipeObject conversion]")
            debug(f"  title={getattr(value, 'title', None)}")
            debug(f"  target={getattr(value, 'target', None)}")
            debug(f"  hash={getattr(value, 'hash', None)}")
            debug(f"  media_kind={getattr(value, 'media_kind', None)}")
            debug(f"  tag={getattr(value, 'tag', None)}")
            debug(f"  tag_summary={getattr(value, 'tag_summary', None)}")
            debug(f"  size_bytes={getattr(value, 'size_bytes', None)}")
            debug(f"  duration_seconds={getattr(value, 'duration_seconds', None)}")
            debug(f"  relationships={getattr(value, 'relationships', None)}")
            debug(f"  url={getattr(value, 'url', None)}")
            debug(
                f"  full_metadata keys={list(getattr(value, 'full_metadata', {}).keys()) if hasattr(value, 'full_metadata') and value.full_metadata else []}"
            )
    except Exception:
        pass

    if isinstance(value, models.PipeObject):
        return value

    known_keys = {
        "hash",
        "store",
        "tag",
        "title",
        "url",
        "source_url",
        "duration",
        "metadata",
        "warnings",
        "path",
        "relationships",
        "is_temp",
        "action",
        "parent_hash",
    }

    # Convert common object-like results into a dict so we can preserve fields like
    # hash/store/url when they come from result tables (e.g., get-url emits UrlItem).
    #
    # Priority:
    # 1) explicit to_dict()
    # 2) best-effort attribute extraction for known PipeObject-ish fields
    if hasattr(value, "to_dict"):
        value = value.to_dict()
    elif not isinstance(value, dict):
        try:
            obj_map: Dict[str,
                          Any] = {}
            for k in (
                    "hash",
                    "store",
                    "provider",
                    "prov",
                    "tag",
                    "title",
                    "url",
                    "source_url",
                    "duration",
                    "duration_seconds",
                    "metadata",
                    "full_metadata",
                    "warnings",
                    "path",
                    "target",
                    "relationships",
                    "is_temp",
                    "action",
                    "parent_hash",
                    "extra",
                    "media_kind",
            ):
                if hasattr(value, k):
                    obj_map[k] = getattr(value, k)
            if obj_map:
                value = obj_map
        except Exception:
            pass

    if isinstance(value, dict):
        # Extract hash and store (canonical identifiers)
        hash_val = value.get("hash")
        store_val = value.get("store") or "PATH"
        if not store_val or store_val == "PATH":
            try:
                extra_store = value.get("extra",
                                        {}).get("store")
            except Exception:
                extra_store = None
            if extra_store:
                store_val = extra_store

        # If no hash, try to compute from path or use placeholder
        if not hash_val:
            path_val = value.get("path")
            if path_val:
                try:
                    from SYS.utils import sha256_file
                    from pathlib import Path

                    hash_val = sha256_file(Path(path_val))
                except Exception:
                    hash_val = "unknown"
            else:
                hash_val = "unknown"

        # Extract title from filename if not provided
        title_val = value.get("title")
        if not title_val:
            path_val = value.get("path")
            if path_val:
                try:
                    from pathlib import Path

                    title_val = Path(path_val).stem
                except Exception:
                    pass

        extra = {
            k: v
            for k, v in value.items() if k not in known_keys
        }

        # Extract URL: prefer direct url field, then url list
        from SYS.metadata import normalize_urls

        url_list = normalize_urls(value.get("url"))
        url_val = url_list[0] if url_list else None
        if len(url_list) > 1:
            extra["url"] = url_list

        # Extract relationships
        rels = value.get("relationships") or {}

        # Canonical tag: accept list or single string
        tag_val: list[str] = []
        if "tag" in value:
            raw_tag = value["tag"]
            if isinstance(raw_tag, list):
                tag_val = [str(t) for t in raw_tag if t is not None]
            elif isinstance(raw_tag, str):
                tag_val = [raw_tag]

        # Consolidate path: prefer explicit path key, but NOT target if it's a URL
        path_val = value.get("path")
        # Only use target as path if it's not a URL (url should stay in url field)
        if not path_val and "target" in value:
            target = value["target"]
            if target and not (isinstance(target,
                                          str) and (target.startswith("http://")
                                                    or target.startswith("https://"))):
                path_val = target

        # If the path value is actually a URL, move it to url_val and clear path_val
        try:
            if isinstance(path_val,
                          str) and (path_val.startswith("http://")
                                    or path_val.startswith("https://")):
                # Prefer existing url_val if present, otherwise move path_val into url_val
                if not url_val:
                    url_val = path_val
                path_val = None
        except Exception:
            pass

        # Extract media_kind if available
        if "media_kind" in value:
            extra["media_kind"] = value["media_kind"]

        pipe_obj = models.PipeObject(
            hash=hash_val,
            store=store_val,
            plugin=str(
                value.get("plugin")
                or value.get("prov")
                or value.get("source")
                or extra.get("plugin")
                or extra.get("source")
                or ""
            ).strip() or None,
            tag=tag_val,
            title=title_val,
            url=url_val,
            source_url=value.get("source_url"),
            duration=value.get("duration") or value.get("duration_seconds"),
            metadata=value.get("metadata") or value.get("full_metadata") or {},
            warnings=list(value.get("warnings") or []),
            path=path_val,
            relationships=rels,
            is_temp=bool(value.get("is_temp",
                                   False)),
            action=value.get("action"),
            parent_hash=value.get("parent_hash"),
            extra=extra,
        )

        return pipe_obj

    # Fallback: build from path argument or bare value
    hash_val = "unknown"
    path_val = default_path or getattr(value, "path", None)
    url_val: Optional[str] = None
    title_val = None

    # If the raw value is a string, treat it as either a URL or a file path.
    # This is important for @-selection results that are plain URL strings.
    if isinstance(value, str):
        s = value.strip()
        if s.lower().startswith(("http://", "https://")):
            url_val = s
            path_val = None
        else:
            path_val = s

    if path_val and path_val != "unknown":
        try:
            from SYS.utils import sha256_file
            from pathlib import Path

            path_obj = Path(path_val)
            hash_val = sha256_file(path_obj)
            # Extract title from filename (without extension)
            title_val = path_obj.stem
        except Exception:
            pass

    # When coming from a raw URL string, mark it explicitly as URL.
    # Otherwise treat it as a local path.
    store_val = "URL" if url_val else "PATH"

    pipe_obj = models.PipeObject(
        hash=hash_val,
        store=store_val,
        plugin=None,
        path=str(path_val) if path_val and path_val != "unknown" else None,
        title=title_val,
        url=url_val,
        source_url=url_val,
        tag=[],
        extra={},
    )

    return pipe_obj


def propagate_metadata(
    previous_items: Sequence[Any],
    new_items: Sequence[Any]
) -> List[Any]:
    """Merge metadata/tags from previous pipeline stage into new items.

    Implements "sticky metadata": items generated by a transformation (download, convert)
    should inherit rich info (lyrics, art, tags) from their source.

    Strategies:
    A. Hash Match: If inputs/outputs share a hash, they are the same item.
    B. Index Match: If lists are same length, assume 1:1 mapping (heuristic).
    C. Explicit Parent: If output has `parent_hash`, link to input with that hash.
    """
    if not previous_items or not new_items:
        return list(new_items)

    try:
        prev_normalized = [coerce_to_pipe_object(p) for p in previous_items]
    except Exception:
        return list(new_items)

    prev_by_hash: Dict[str, models.PipeObject] = {}
    for p_obj in prev_normalized:
        if p_obj.hash and p_obj.hash != "unknown":
            prev_by_hash[p_obj.hash] = p_obj

    normalized: List[Any] = []
    
    # Pre-calculate length matching for heuristic
    is_same_length = len(new_items) == len(prev_normalized)

    for i, item in enumerate(new_items):
        if isinstance(item, dict) and item.get("_skip_metadata_propagation"):
            normalized.append(item)
            continue
        try:
            obj = coerce_to_pipe_object(item)
        except Exception:
            normalized.append(item) # Should not happen given coerce guards
            continue

        parent: Optional[models.PipeObject] = None

        # Strategy A: Precise Hash Match
        if obj.hash in prev_by_hash:
            parent = prev_by_hash[obj.hash]

        # Strategy B: Index Match (Heuristic)
        if not parent and is_same_length:
            parent = prev_normalized[i]

        # Strategy C: Explicit Parent Hash
        if not parent and obj.parent_hash and obj.parent_hash in prev_by_hash:
            parent = prev_by_hash[obj.parent_hash]

        if parent:
            # 1. Tags: Merge unique tags
            if parent.tag:
                if not obj.tag:
                    obj.tag = list(parent.tag)
                else:
                    curr_tags = {str(t).lower() for t in obj.tag}
                    for pt in parent.tag:
                        if str(pt).lower() not in curr_tags:
                            obj.tag.append(pt)

            # 2. Metadata: Merge missing keys
            if parent.metadata:
                if not obj.metadata:
                    obj.metadata = parent.metadata.copy()
                else:
                    for mk, mv in parent.metadata.items():
                        if mk not in obj.metadata:
                            obj.metadata[mk] = mv

            # 3. Source URL: Propagate if missing
            if parent.source_url and not obj.source_url:
                obj.source_url = parent.source_url
            elif parent.url and not obj.source_url and not obj.url:
                # If parent had a URL and child has none, it's likely the source
                obj.source_url = parent.url

            # 4. Relationships: Merge missing keys
            if parent.relationships:
                if not obj.relationships:
                    obj.relationships = parent.relationships.copy()
                else:
                    for rk, rv in parent.relationships.items():
                        if rk not in obj.relationships:
                            obj.relationships[rk] = rv

            # 5. Extra (Notes/etc): Merge missing keys
            # Important for passing 'notes' payload (lyrics, captions)
            if parent.extra:
                if not obj.extra:
                    obj.extra = parent.extra.copy()
                else:
                    # Recursive merge for 'notes' dict specifically?
                    # For now just shallow merge keys, but handle 'notes' specially if valid.
                    for ek, ev in parent.extra.items():
                        if ek not in obj.extra:
                            obj.extra[ek] = ev
                        elif ek == "notes" and isinstance(ev, dict) and isinstance(obj.extra[ek], dict):
                            # Merge notes dict
                            for nk, nv in ev.items():
                                if nk not in obj.extra[ek]:
                                    obj.extra[ek][nk] = nv

        normalized.append(obj)

    return normalized


def register_url_with_local_library(
    pipe_obj: models.PipeObject,
    config: Dict[str,
                 Any]
) -> bool:
    """Register url with a file in the local library database.

    This is called automatically by download cmdlet to ensure url are persisted
    without requiring a separate add-url step in the pipeline.

    Args:
        pipe_obj: PipeObject with path and url
        config: Config dict containing local library path

    Returns:
        True if url were registered, False otherwise
    """
    # Folder store removed; local library URL registration is disabled.
    return False
def check_url_exists_in_storage(
    urls: Sequence[str],
    storage: Any,
    hydrus_available: bool,
    final_output_dir: Optional[Path] = None,
    *,
    auto_continue_duplicates: bool = True,
    force_prompt_in_pipeline: bool = False,
) -> bool:
    """Pre-flight check to see if URLs already exist in storage.

    Args:
        urls: List of URLs to check
        storage: The storage interface
        hydrus_available: Whether Hydrus is available
        final_output_dir: Final output directory (to skip if same as storage)

    Returns:
        True if check passed (user said yes or no dups), False if user said no (stop).
    """
    if storage is None:
        debug("Bulk URL preflight skipped: storage unavailable")
        return True

    try:
        current_cmd_text = pipeline_context.get_current_command_text("")
    except Exception:
        current_cmd_text = ""

    try:
        stage_ctx = pipeline_context.get_stage_context()
    except Exception:
        stage_ctx = None

    in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or "")))
    start_time = time.monotonic()
    time_budget = 45.0
    if in_pipeline:
        try:
            already_checked = bool(
                pipeline_context.load_value(
                    "preflight.url_duplicates.checked", default=False
                )
            )
        except Exception:
            already_checked = False

        if already_checked:
            debug("Bulk URL preflight: already checked in pipeline; skipping duplicate check")
            return True

    def _load_preflight_cache() -> Dict[str, Any]:
        try:
            existing = pipeline_context.load_value("preflight", default=None)
        except Exception:
            existing = None
        return existing if isinstance(existing, dict) else {}

    def _store_preflight_cache(cache: Dict[str, Any]) -> None:
        try:
            pipeline_context.store_value("preflight", cache)
        except Exception:
            pass

    def _mark_preflight_checked() -> None:
        if not in_pipeline:
            return
        try:
            pipeline_context.store_value("preflight.url_duplicates.checked", True)
        except Exception:
            pass
        preflight_cache = _load_preflight_cache()
        preflight_cache["url_duplicates_checked"] = True
        url_dup_cache = preflight_cache.get("url_duplicates")
        if not isinstance(url_dup_cache, dict):
            url_dup_cache = {}
        url_dup_cache["checked"] = True
        preflight_cache["url_duplicates"] = url_dup_cache
        _store_preflight_cache(preflight_cache)

    def _timed_out(reason: str) -> bool:
        try:
            if (time.monotonic() - start_time) >= time_budget:
                debug(
                    f"Bulk URL preflight timed out after {time_budget:.0f}s ({reason}); continuing"
                )
                _mark_preflight_checked()
                return True
        except Exception:
            return False
        return False

    if in_pipeline and auto_continue_duplicates:
        try:
            cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="")
            cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None)
        except Exception:
            cached_cmd = ""
            cached_decision = None

        if (not force_prompt_in_pipeline) and cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""):
            _mark_preflight_checked()
            if bool(cached_decision):
                return True
            try:
                pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
            except Exception:
                pass
            return False

    unique_urls: List[str] = []
    for u in urls or []:
        s = str(u or "").strip()
        if s and s not in unique_urls:
            unique_urls.append(s)
    if len(unique_urls) == 0:
        return True

    try:
        from SYS.metadata import normalize_urls
    except Exception:
        normalize_urls = None  # type: ignore[assignment]

    def _httpish(value: str) -> bool:
        try:
            return bool(value) and (value.startswith("http://") or value.startswith("https://"))
        except Exception:
            return False

    def _normalize_url_for_search(value: str) -> str:
        url = str(value or "").strip()

        # Strip fragment (e.g., #t=10) before matching
        url = url.split("#", 1)[0]

        # Strip common time/tracking query params for matching
        try:
            parsed = urlparse(url)
        except Exception:
            parsed = None

        if parsed is not None and parsed.query:
            time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"}
            tracking_prefixes = ("utm_",)
            try:
                pairs = parse_qsl(parsed.query, keep_blank_values=True)
                filtered = []
                for key, val in pairs:
                    key_norm = str(key or "").lower()
                    if key_norm in time_keys:
                        continue
                    if key_norm.startswith(tracking_prefixes):
                        continue
                    filtered.append((key, val))
                if filtered:
                    url = urlunparse(parsed._replace(query=urlencode(filtered, doseq=True)))
                else:
                    url = urlunparse(parsed._replace(query=""))
            except Exception:
                pass

        # Remove protocol (http://, https://, ftp://, etc.)
        url = re.sub(r"^[a-z][a-z0-9+.-]*://", "", url, flags=re.IGNORECASE)

        # Remove www. prefix (case-insensitive)
        url = re.sub(r"^www\.", "", url, flags=re.IGNORECASE)

        return url.lower()

    def _expand_url_variants(value: str) -> List[str]:
        if not _httpish(value):
            return []

        try:
            parsed = urlparse(value)
        except Exception:
            return []

        if parsed.scheme.lower() not in {"http", "https"}:
            return []

        out: List[str] = []

        def _add_variant(candidate: str) -> None:
            _maybe_add(candidate)
            try:
                lower = str(candidate or "").lower()
            except Exception:
                lower = ""
            if lower and lower != candidate:
                _maybe_add(lower)

            try:
                parsed_candidate = urlparse(candidate)
            except Exception:
                parsed_candidate = None

            if parsed_candidate is None:
                return

            host = (parsed_candidate.hostname or "").strip().lower()
            if host.startswith("www."):
                host = host[4:]
                if host:
                    netloc = host
                    try:
                        if parsed_candidate.port:
                            netloc = f"{netloc}:{parsed_candidate.port}"
                    except Exception:
                        pass
                    try:
                        if parsed_candidate.username or parsed_candidate.password:
                            userinfo = parsed_candidate.username or ""
                            if parsed_candidate.password:
                                userinfo = f"{userinfo}:{parsed_candidate.password}"
                            if userinfo:
                                netloc = f"{userinfo}@{netloc}"
                    except Exception:
                        pass
                    alt = urlunparse(parsed_candidate._replace(netloc=netloc))
                    _maybe_add(alt)
                    try:
                        lower_alt = alt.lower()
                    except Exception:
                        lower_alt = ""
                    if lower_alt and lower_alt != alt:
                        _maybe_add(lower_alt)

        def _maybe_add(candidate: str) -> None:
            if not candidate or candidate == value:
                return
            if candidate not in out:
                out.append(candidate)

        if parsed.fragment:
            _add_variant(urlunparse(parsed._replace(fragment="")))

        time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"}
        tracking_prefixes = ("utm_",)

        try:
            query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
        except Exception:
            query_pairs = []

        if query_pairs or parsed.fragment:
            filtered_pairs = []
            removed = False
            for key, val in query_pairs:
                key_norm = str(key or "").lower()
                if key_norm in time_keys:
                    removed = True
                    continue
                if key_norm.startswith(tracking_prefixes):
                    removed = True
                    continue
                filtered_pairs.append((key, val))

            if removed:
                new_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else ""
                _add_variant(urlunparse(parsed._replace(query=new_query, fragment="")))

        return out

    def _dedupe_needles(raw_needles: Sequence[str]) -> List[str]:
        output: List[str] = []
        seen: set[str] = set()
        for candidate in (raw_needles or []):
            candidate_text = str(candidate or "").strip()
            if not candidate_text:
                continue
            key = candidate_text.lower()
            if key in seen:
                continue
            seen.add(key)
            output.append(candidate_text)
        return output

    url_needles: Dict[str, List[str]] = {}
    for u in unique_urls:
        needles: List[str] = []
        if normalize_urls is not None:
            try:
                needles.extend([n for n in (normalize_urls(u) or []) if isinstance(n, str)])
            except Exception:
                needles = []
        if not needles:
            needles = [u]
        filtered: List[str] = []
        for n in needles:
            n2 = str(n or "").strip()
            if not n2:
                continue
            if not _httpish(n2):
                continue
            if n2 not in filtered:
                filtered.append(n2)
        lowered: List[str] = []
        for n2 in filtered:
            try:
                lower = n2.lower()
            except Exception:
                lower = ""
            if lower and lower != n2 and lower not in filtered and lower not in lowered:
                lowered.append(lower)
        normalized: List[str] = []
        for n2 in filtered:
            norm = _normalize_url_for_search(n2)
            if norm and norm not in normalized and norm not in filtered:
                normalized.append(norm)
        expanded: List[str] = []
        for n2 in filtered:
            for extra in _expand_url_variants(n2):
                if extra not in expanded and extra not in filtered and extra not in lowered:
                    expanded.append(extra)
                norm_extra = _normalize_url_for_search(extra)
                if (
                    norm_extra
                    and norm_extra not in normalized
                    and norm_extra not in filtered
                    and norm_extra not in expanded
                    and norm_extra not in lowered
                ):
                    normalized.append(norm_extra)

        combined = filtered + expanded + lowered + normalized
        deduped = _dedupe_needles(combined)
        url_needles[u] = deduped if deduped else [u]

    if in_pipeline:
        preflight_cache = _load_preflight_cache()
        url_dup_cache = preflight_cache.get("url_duplicates")
        if not isinstance(url_dup_cache, dict):
            url_dup_cache = {}
        cached_urls = url_dup_cache.get("urls")
        cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()

        if cached_set:
            all_cached = True
            for original_url, needles in url_needles.items():
                original_cached = str(original_url or "") in cached_set
                needles_cached = True
                if original_cached:
                    for needle in (needles or []):
                        needle_text = str(needle or "")
                        if not needle_text:
                            continue
                        if needle_text not in cached_set:
                            needles_cached = False
                            break
                else:
                    needles_cached = False

                if original_cached and needles_cached:
                    continue

                all_cached = False
                break

            if all_cached:
                debug("Bulk URL preflight: cached for pipeline; skipping duplicate check")
                _mark_preflight_checked()
                return True

    if _timed_out("before backend scan"):
        return True

    # Use bulk mode only if we have a significant number of URLs.
    # For small sets (1-3 URLs), individual targeted searches are faster
    # and more accurate than scanning all files with URLs in the backend.
    bulk_mode = len(unique_urls) > 3

    def _build_bulk_patterns(needles_map: Dict[str, List[str]], max_per_url: int = 3, max_total: int = 240) -> List[str]:
        patterns: List[str] = []
        for _original, needles in needles_map.items():
            for needle in (needles or [])[:max_per_url]:
                needle_text = str(needle or "").strip()
                if not needle_text:
                    continue
                if needle_text not in patterns:
                    patterns.append(needle_text)
                if len(patterns) >= max_total:
                    return patterns
        return patterns

    bulk_patterns = _build_bulk_patterns(url_needles)

    def _match_normalized_url(pattern_text: str, candidate_url: str) -> bool:
        pattern_norm = _normalize_url_for_search(pattern_text)
        candidate_norm = _normalize_url_for_search(candidate_url)
        if not pattern_norm or not candidate_norm:
            return False
        if pattern_norm == candidate_norm:
            return True
        return pattern_norm in candidate_norm

    def _extract_urls_from_hit(
        hit: Any,
        backend: Any,
        *,
        allow_backend_lookup: bool = True,
    ) -> List[str]:
        url_values: List[str] = []
        try:
            raw_urls = get_field(hit, "known_urls") or get_field(hit, "urls") or get_field(hit, "url")
            if isinstance(raw_urls, str) and raw_urls.strip():
                url_values.append(raw_urls.strip())
            elif isinstance(raw_urls, (list, tuple, set)):
                for item in raw_urls:
                    if isinstance(item, str) and item.strip():
                        url_values.append(item.strip())
        except Exception:
            url_values = []

        if url_values or not allow_backend_lookup:
            return url_values

        try:
            file_hash = get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or ""
        except Exception:
            file_hash = ""

        if file_hash:
            try:
                fetched = backend.get_url(str(file_hash))
                if isinstance(fetched, str) and fetched.strip():
                    url_values.append(fetched.strip())
                elif isinstance(fetched, (list, tuple, set)):
                    for item in fetched:
                        if isinstance(item, str) and item.strip():
                            url_values.append(item.strip())
            except Exception:
                pass

        return url_values

    def _build_display_row_for_hit(
        hit: Any,
        backend_name: str,
        original_url: str,
    ) -> Dict[str, Any]:
        try:
            from SYS.result_table import build_display_row
            extracted = build_display_row(hit, keys=["title", "store", "hash", "ext", "size"])
        except Exception:
            extracted = {}

        try:
            title = extracted.get("title") or get_field(hit, "title") or get_field(hit, "name") or get_field(hit, "target") or get_field(hit, "path") or "(exists)"
        except Exception:
            title = "(exists)"

        try:
            file_hash = extracted.get("hash") or get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or ""
        except Exception:
            file_hash = ""

        ext = extracted.get("ext") if isinstance(extracted, dict) else ""
        size_val = extracted.get("size") if isinstance(extracted, dict) else None

        return build_table_result_payload(
            title=str(title),
            columns=[
                ("Title", str(title)),
                ("Store", str(get_field(hit, "store") or backend_name)),
                ("Hash", str(file_hash or "")),
                ("Ext", str(ext or "")),
                ("Size", size_val),
                ("URL", original_url),
            ],
            store=str(get_field(hit, "store") or backend_name),
            hash=str(file_hash or ""),
            ext=str(ext or ""),
            size=size_val,
            url=original_url,
        )

    def _search_backend_url_hits(
        backend: Any,
        backend_name: str,
        original_url: str,
        needles: Sequence[str],
    ) -> Optional[Dict[str, Any]]:
        backend_hits: List[Dict[str, Any]] = []

        # 1) Try exact match first (no wildcards).
        # This is extremely fast for Hydrus and others that support direct URL lookup.
        for needle in (needles or [])[:5]:
            needle_stripped = str(needle or "").strip()
            if not needle_stripped or not _httpish(needle_stripped):
                continue
            try:
                # Use 'url:' prefix to ensure storage layers (like Hydrus) recognize it as a URL lookup
                query = f"url:{needle_stripped}"
                backend_hits = backend.search(query, limit=1, minimal=True) or []
                if backend_hits:
                    return _build_display_row_for_hit(backend_hits[0], backend_name, original_url)
            except Exception:
                continue

        # 2) Fallback to wildcard substring search for normalized variants.
        # This is for backends where the URL might be stored differently (partial match).
        for needle in (needles or [])[:3]:
            needle_text = str(needle or "").strip()
            if not needle_text:
                continue
            search_needle = _normalize_url_for_search(needle_text) or needle_text
            query = f"url:*{search_needle}*"
            try:
                backend_hits = backend.search(query, limit=1, minimal=True) or []
                if backend_hits:
                    break
            except Exception:
                continue

        if not backend_hits:
            return None

        hit = backend_hits[0]
        return _build_display_row_for_hit(hit, backend_name, original_url)

    backend_names: List[str] = []
    try:
        backend_names_all = storage.list_searchable_backends()
    except Exception:
        backend_names_all = []

    for backend_name in backend_names_all:
        try:
            backend = storage[backend_name]
        except Exception:
            continue

        try:
            if str(backend_name).strip().lower() == "temp":
                continue
        except Exception:
            pass

        try:
            backend_location = getattr(backend, "_location", None)
            if backend_location and final_output_dir:
                backend_path = Path(str(backend_location)).expanduser().resolve()
                temp_path = Path(str(final_output_dir)).expanduser().resolve()
                if backend_path == temp_path:
                    continue
        except Exception:
            pass

        backend_names.append(backend_name)

    if not backend_names:
        debug("Bulk URL preflight skipped: no searchable backends")
        return True

    try:
        debug_panel(
            "URL preflight",
            [
                ("url_count", len(unique_urls)),
                ("pipeline", in_pipeline),
                ("bulk_mode", bulk_mode),
                ("backends", ", ".join(str(name) for name in backend_names)),
            ],
            border_style="yellow",
        )
    except Exception:
        pass

    seen_pairs: set[tuple[str, str]] = set()
    matched_urls: set[str] = set()
    match_rows: List[Dict[str, Any]] = []
    max_rows = 200

    hydrus_provider = None
    try:
        from PluginCore.registry import get_plugin

        hydrus_provider = get_plugin("hydrusnetwork", config)
    except Exception:
        hydrus_provider = None

    for backend_name in backend_names:
        if _timed_out("backend scan"):
            return True
        if len(match_rows) >= max_rows:
            break
        try:
            backend = storage[backend_name]
        except Exception:
            continue

        is_hydrus_backend = False
        try:
            is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
        except Exception:
            is_hydrus_backend = False
        if not is_hydrus_backend:
            try:
                is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork"
            except Exception:
                is_hydrus_backend = False

        if is_hydrus_backend:
            if not hydrus_available:
                debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")

            if _timed_out("hydrus scan"):
                return True

            for original_url, needles in url_needles.items():
                if _timed_out("hydrus per-url scan"):
                    return True
                if len(match_rows) >= max_rows:
                    break
                if (original_url, str(backend_name)) in seen_pairs:
                    continue

                found_hash: Optional[str] = None
                found = False
                lookup_exact = getattr(backend, "find_hashes_by_url", None)
                if callable(lookup_exact):
                    for needle in [original_url, *(needles or [])][:7]:
                        needle_text = str(needle or "").strip()
                        if not _httpish(needle_text):
                            continue
                        try:
                            exact_hashes = lookup_exact(needle_text) or []
                        except Exception:
                            continue
                        if not isinstance(exact_hashes, list) or not exact_hashes:
                            continue
                        try:
                            found_hash = str(exact_hashes[0] or "").strip().lower()
                        except Exception:
                            found_hash = None
                        found = True
                        break

                if not found:
                    continue

                seen_pairs.add((original_url, str(backend_name)))
                matched_urls.add(original_url)
                display_row = build_table_result_payload(
                    title="(exists)",
                    columns=[
                        ("Title", "(exists)"),
                        ("Store", str(backend_name)),
                        ("Hash", found_hash or ""),
                        ("URL", original_url),
                    ],
                    store=str(backend_name),
                    hash=found_hash or "",
                    url=original_url,
                )
                match_rows.append(display_row)
            continue

        if bulk_mode and bulk_patterns:
            bulk_hits: Optional[List[Any]] = None
            bulk_limit = min(2000, max(200, len(unique_urls) * 8))
            try:
                bulk_hits = backend.search(
                    "url:*",
                    limit=bulk_limit,
                    pattern_hint=bulk_patterns,
                ) or []
            except Exception:
                try:
                    bulk_hits = backend.search("url:*", limit=bulk_limit) or []
                except Exception:
                    bulk_hits = None

            if bulk_hits is not None:
                for hit in bulk_hits:
                    if _timed_out("backend bulk scan"):
                        return True
                    if len(match_rows) >= max_rows:
                        break
                    url_values = _extract_urls_from_hit(hit, backend, allow_backend_lookup=False)
                    if not url_values:
                        continue

                    for original_url, needles in url_needles.items():
                        if _timed_out("backend bulk scan"):
                            return True
                        if len(match_rows) >= max_rows:
                            break
                        if (original_url, str(backend_name)) in seen_pairs:
                            continue

                        matched = False
                        for url_value in url_values:
                            for needle in (needles or []):
                                if _match_normalized_url(str(needle or ""), str(url_value or "")):
                                    matched = True
                                    break
                            if matched:
                                break

                        if not matched:
                            continue

                        seen_pairs.add((original_url, str(backend_name)))
                        matched_urls.add(original_url)
                        match_rows.append(
                            _build_display_row_for_hit(hit, str(backend_name), original_url)
                        )
                continue

        for original_url, needles in url_needles.items():
            if _timed_out("backend per-url scan"):
                return True
            if len(match_rows) >= max_rows:
                break
            if (original_url, str(backend_name)) in seen_pairs:
                continue

            display_row = _search_backend_url_hits(backend, str(backend_name), original_url, needles)
            if not display_row:
                continue

            seen_pairs.add((original_url, str(backend_name)))
            matched_urls.add(original_url)
            match_rows.append(display_row)

    if not match_rows:
        if in_pipeline:
            preflight_cache = _load_preflight_cache()
            url_dup_cache = preflight_cache.get("url_duplicates")
            if not isinstance(url_dup_cache, dict):
                url_dup_cache = {}

            cached_urls = url_dup_cache.get("urls")
            cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()

            for original_url, needles in url_needles.items():
                cached_set.add(original_url)
                for needle in needles or []:
                    cached_set.add(str(needle))

            url_dup_cache["urls"] = sorted(cached_set)
            preflight_cache["url_duplicates"] = url_dup_cache
            _store_preflight_cache(preflight_cache)
            _mark_preflight_checked()
        return True

    table = Table(f"URL already exists ({len(matched_urls)} url(s))", max_columns=10)
    table._interactive(True)
    try:
        table._perseverance(True)
    except Exception:
        pass

    for row in match_rows:
        table.add_result(row)

    try:
        pipeline_context.set_last_result_table_overlay(table, match_rows)
    except Exception:
        pass

    suspend = getattr(pipeline_context, "suspend_live_progress", None)
    cm: AbstractContextManager[Any] = nullcontext()
    if callable(suspend):
        try:
            maybe_cm = suspend()
            if maybe_cm is not None:
                cm = maybe_cm  # type: ignore[assignment]
        except Exception:
            cm = nullcontext()

    auto_confirm_reason: Optional[str] = None
    if in_pipeline and stage_ctx is not None:
        try:
            total_stages = int(getattr(stage_ctx, "total_stages", 0))
        except Exception:
            total_stages = 0
        try:
            is_last_stage = bool(getattr(stage_ctx, "is_last_stage", False))
        except Exception:
            is_last_stage = False
        if total_stages > 1 and not is_last_stage and not force_prompt_in_pipeline:
            auto_confirm_reason = "pipeline stage (pre-last)"
    if auto_confirm_reason is None:
        try:
            stdin_interactive = bool(sys.stdin and sys.stdin.isatty())
        except Exception:
            stdin_interactive = False
        if not stdin_interactive:
            auto_confirm_reason = "non-interactive stdin"

    answered_yes = True
    auto_declined = False
    with cm:
        get_stderr_console().print(table)
        setattr(table, "_rendered_by_cmdlet", True)
        if auto_confirm_reason is None:
            answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console()))
        else:
            answered_yes = bool(auto_continue_duplicates)
            auto_declined = not answered_yes
            if answered_yes:
                debug(
                    f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
                )
                try:
                    log(
                        f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
                        file=sys.stderr,
                    )
                except Exception:
                    pass
            else:
                debug(
                    f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input."
                )
                try:
                    log(
                        f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.",
                        file=sys.stderr,
                    )
                except Exception:
                    pass

        if in_pipeline and auto_continue_duplicates:
            try:
                existing = pipeline_context.load_value("preflight", default=None)
            except Exception:
                existing = None
            preflight_cache: Dict[str, Any] = existing if isinstance(existing, dict) else {}
            url_dup_cache = preflight_cache.get("url_duplicates")
            if not isinstance(url_dup_cache, dict):
                url_dup_cache = {}
            url_dup_cache["command"] = str(current_cmd_text or "")
            url_dup_cache["continue"] = bool(answered_yes)
            cached_urls = url_dup_cache.get("urls")
            cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set()
            for original_url, needles in url_needles.items():
                cached_set.add(original_url)
                for needle in needles or []:
                    cached_set.add(str(needle))
            url_dup_cache["urls"] = sorted(cached_set)
            preflight_cache["url_duplicates"] = url_dup_cache
            try:
                pipeline_context.store_value("preflight", preflight_cache)
            except Exception:
                pass

        if not answered_yes:
            if in_pipeline and not auto_declined:
                 try:
                    pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
                 except Exception:
                    pass
            _mark_preflight_checked()
            return False
        _mark_preflight_checked()
    return True


def display_and_persist_items(
    items: List[Any],
    title: str = "Result",
    subject: Optional[Any] = None,
    display_type: str = "item",
    table: Optional[Table] = None,
) -> None:
    """Display detail panels and persist items for @N selection.
    
    This helper function:
    1. Renders individual detail panels for each item
    2. Creates a result table with all items (or uses provided table)
    3. Persists the table so @N selection works across command boundaries
    
    Args:
        items: List of items/dicts to display and make selectable
        title: Title for the result table (default: "Result")
        subject: Optional subject to associate with the results (usually first item or list)
        display_type: Type of display - "item" (default), "tag", or "custom" (no panels)
        table: Optional pre-built Table object (if None, creates new Table with title)
    """
    if not items:
        return
    
    try:
        # Create result table if not provided
        if table is None:
            table = Table(title=title)
        
        # Render detail panels for each item (unless display_type is "custom")
        if display_type != "custom":
            try:
                from SYS.rich_display import render_item_details_panel
                
                # Determine panel title prefix based on display type
                panel_prefix = "Tag" if display_type == "tag" else "Item"
                
                # Render detail panel for each item
                for idx, item in enumerate(items, 1):
                    try:
                        render_item_details_panel(item, title=f"#{idx} {panel_prefix} Details")
                    except Exception:
                        pass
            except Exception:
                pass
        
        # Add items to table if not already added by caller
        if not getattr(table, "_items_added", False):
            for item in items:
                table.add_result(item)
        
        setattr(table, "_rendered_by_cmdlet", True)
        
        # Persist table for @N selection across command boundaries
        publish_result_table(pipeline_context, table, items, subject=subject)
    except Exception:
        pass
