diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..dc71b2d --- /dev/null +++ b/.env.example @@ -0,0 +1,115 @@ +# ============================================================================= +# Medeia-Macina Environment Variables +# ============================================================================= +# Copy this file to .env and adjust values as needed. +# +# All variables are optional unless noted otherwise. The application auto-detects +# sensible defaults for most settings. + +# --------------------------------------------------------------------------- +# Core / Debug +# --------------------------------------------------------------------------- + +# Enable debug mode (verbose logging, diagnostic output, rich tracebacks). +# Set to "1" to enable. +# Default: disabled (unless the database config key `debug` is "1"/"true"/"yes"/"on") +MM_DEBUG=0 + +# Enable raw HTTP request/response body logging in debug output. When set to +# "0", "false", "no", or "off", raw bodies are suppressed even while MM_DEBUG +# is active. Any other value (or unset) follows MM_DEBUG. +MM_HTTP_RAW=0 + +# --------------------------------------------------------------------------- +# Repository / Paths +# --------------------------------------------------------------------------- + +# Application root directory. Searched for medios.db, CLI.py, and config.conf. +# Precedence: MM_REPO > MM_ROOT > REPO > auto-detection from CWD. +MM_REPO= + +# Alternative root directory name (checked if MM_REPO is not set). +MM_ROOT= + +# Legacy / generic repository path (checked after MM_REPO and MM_ROOT). +REPO= + +# Semicolon-separated list of external plugin directories. Plugins placed in +# these directories are discovered and registered at startup. +# Example: C:\extra-plugins;C:\more-plugins +MM_PLUGIN_PATH= + +# Legacy alias for MM_PLUGIN_PATH (checked second). +MEDEIA_PLUGIN_PATH= + +# --------------------------------------------------------------------------- +# Networking +# --------------------------------------------------------------------------- + +# Custom SSL/TLS CA certificate bundle path (PEM format). +# Overrides system-wide trust stores for all TLS connections made by the +# application. Useful behind corporate MITM proxies or custom CAs. +SSL_CERT_FILE= + +# --------------------------------------------------------------------------- +# MPV Integration +# --------------------------------------------------------------------------- + +# Override the default MPV IPC socket / named-pipe path. Used by the CLI, +# the pipeline helper, and the lyrics overlay to communicate with mpv. +# Default (Windows): \\.\pipe\mpv-medios-macina +# Default (macOS/Linux): /tmp/mpv-medios-macina.sock +MEDEIA_MPV_IPC= + +# Legacy alias for MEDEIA_MPV_IPC (checked second). +MPV_IPC_SERVER= + +# Minimum log level for mpv's internal message stream forwarded to the helper +# log. Valid values: "fatal", "error", "warn" (default), "info", "debug", "v", +# "trace". +MEDEIA_MPV_HELPER_MPVLOG=warn + +# --------------------------------------------------------------------------- +# Dependencies / Updates +# --------------------------------------------------------------------------- + +# Automatically update Python dependencies on client startup. +# Set to "1" to force `pip install --upgrade` on the requirements file. +MM_UPDATE_DEPS=0 + +# --------------------------------------------------------------------------- +# Playwright / Browser Automation +# --------------------------------------------------------------------------- + +# Path to the ffmpeg executable. Used by Playwright for video recording and +# processing. Resolution order: +# 1. Plugin config key `plugin.playwright.ffmpeg_path` +# 2. Environment variable `FFMPEG_PATH` +# 3. Environment variable `PLAYWRIGHT_FFMPEG_PATH` (legacy) +# 4. Project-bundled ffmpeg (MPV/ffmpeg/bin/) +# 5. System PATH lookup (`which ffmpeg`) +FFMPEG_PATH= + +# Legacy Playwright-specific ffmpeg path. Prefer `FFMPEG_PATH` instead. +PLAYWRIGHT_FFMPEG_PATH= + +# --------------------------------------------------------------------------- +# Internet Archive +# --------------------------------------------------------------------------- + +# Internet Archive S3-like API access key. Used for uploads. Can also be +# configured in the plugin section of the config (key: `access_key`). +IA_ACCESS_KEY= + +# Internet Archive S3-like API secret key. Used for uploads. Can also be +# configured in the plugin section of the config (key: `secret_key`). +IA_SECRET_KEY= + +# --------------------------------------------------------------------------- +# Soulseek +# --------------------------------------------------------------------------- + +# Enable verbose aioslsk library output (normally suppressed). +# Set to "1" to see debug-level noise from the Soulseek networking layer. +DOWNLOAD_SOULSEEK_VERBOSE=0 +# Note: The old name `DOWNLOW_SOULSEEK_VERBOSE` (typo) is deprecated. diff --git a/.gitignore b/.gitignore index 0c3b714..3297651 100644 --- a/.gitignore +++ b/.gitignore @@ -82,8 +82,6 @@ target/ # IPython profile_default/ ipython_config.py -config.conf -config.d/ # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: @@ -162,13 +160,6 @@ dmypy.json # Cython debug symbols cython_debug/ -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - # Ruff stuff: .ruff_cache/ @@ -205,7 +196,6 @@ luac.out # Shared objects (inc. Windows DLLs) *.dll -*.so *.so.* *.dylib @@ -218,12 +208,11 @@ luac.out *.hex -config.conf -config.d/ MPV/ffmpeg/* +config_backups/ + Log/ -Log/medeia_macina/telegram.session mpv_logs_with_db.txt *.session example.py @@ -232,21 +221,19 @@ MPV/portable_config/watch_later* hydrusnetwork .style.yapf .yapfignore -tests/ scripts/mm.ps1 scripts/mm ..style.yapf .yapfignore tmp_* *.secret -authtoken.secret mypy. .idea medios.db medios* mypy.ini -\logs\* +logs/* logs* logs.db logs.db* diff --git a/API/base.py b/API/base.py index 02c23b7..01fb236 100644 --- a/API/base.py +++ b/API/base.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import Any, Dict, Optional +import atexit import threading from .HTTP import HTTPClient @@ -19,6 +20,7 @@ class API: self.timeout = float(timeout) self._http_client: Optional[HTTPClient] = None self._http_client_lock = threading.Lock() + atexit.register(self.close) def _get_http_client(self) -> HTTPClient: """Return a reusable opened HTTP client for this API instance.""" @@ -49,12 +51,6 @@ class API: pass self._http_client = None - def __del__(self) -> None: - try: - self.close() - except Exception: - pass - def _get_json( self, path: str, diff --git a/API/cmdlet.py b/API/cmdlet.py index 4a5f1a1..12d1542 100644 --- a/API/cmdlet.py +++ b/API/cmdlet.py @@ -69,7 +69,7 @@ def run_cmdlet( ) -> CmdletRunResult: """Run a single cmdlet programmatically and return structured results. - This is intended for TUI/webapp consumers that want cmdlet behavior without + This is intended for programmatic consumers that want cmdlet behavior without going through the interactive CLI loop. Notes: diff --git a/CLI.py b/CLI.py index 99aa724..2442b8a 100644 --- a/CLI.py +++ b/CLI.py @@ -827,9 +827,14 @@ class CmdletCompleter(Completer): return None raw_plugin = CmdletCompleter._flag_value(stage_tokens, "-plugin", "--plugin") if raw_plugin: - # Strip quotes if present, then normalize to lowercase stripped = CmdletCompleter._strip_quotes(str(raw_plugin or "")) - return stripped.strip().lower() if stripped else None + if not stripped: + return None + plugin_name = stripped.strip().lower() + if "," in plugin_name: + parts = [p.strip() for p in plugin_name.split(",") if p.strip()] + return parts[0] if parts else None + return plugin_name return None @staticmethod @@ -1058,11 +1063,20 @@ class CmdletCompleter(Completer): logical: idx for idx, logical in enumerate(file_stage_order) } + action_logicals = {logical for logical, _ in self._FILE_STAGE_ACTION_CMDLETS} + chosen_actions: Set[str] = set() + for token in stage_tokens: + logical = str(token).lstrip("-").strip().lower() + if logical in action_logicals: + chosen_actions.add(logical) + filtered: List[Tuple[int, int, str]] = [] for index, arg in enumerate(arg_names): logical = str(arg or "").lstrip("-").strip().lower() if file_stage_order is not None and logical not in file_stage_rank: continue + if chosen_actions and logical in action_logicals and logical not in chosen_actions: + continue if logical == "open": continue if logical == "instance": @@ -3085,6 +3099,10 @@ Come to love it when others take what you share, as there is no greater joy if "|" in tokens or (tokens and tokens[0].startswith("@")): self._pipeline_executor.execute_tokens(tokens) else: + try: + ctx.clear_pending_pipeline_tail() + except Exception: + pass cmd_name = tokens[0].replace("_", "-").lower() is_help = any( arg in {"-help", diff --git a/PluginCore/registry.py b/PluginCore/registry.py index c592b19..1d0d1d3 100644 --- a/PluginCore/registry.py +++ b/PluginCore/registry.py @@ -330,36 +330,43 @@ class PluginRegistry: for plugin_dir in _iter_external_plugin_dirs(): if self._is_builtin_package_dir(plugin_dir): continue + + plugin_dir_str = str(plugin_dir) + path_added = False + if plugin_dir_str and plugin_dir_str not in sys.path: + sys.path.insert(0, plugin_dir_str) + path_added = True + try: - plugin_dir_str = str(plugin_dir) - if plugin_dir_str and plugin_dir_str not in sys.path: - sys.path.insert(0, plugin_dir_str) - except Exception: - pass + for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir): + if module_name in self._external_modules: + continue - for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir): - if module_name in self._external_modules: - continue + try: + if is_package: + spec = importlib.util.spec_from_file_location( + module_name, + str(module_path), + submodule_search_locations=[str(module_path.parent)], + ) + else: + spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + if spec is None or spec.loader is None: + raise ImportError("missing module spec loader") - try: - if is_package: - spec = importlib.util.spec_from_file_location( - module_name, - str(module_path), - submodule_search_locations=[str(module_path.parent)], - ) - else: - spec = importlib.util.spec_from_file_location(module_name, str(module_path)) - if spec is None or spec.loader is None: - raise ImportError("missing module spec loader") - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - self._external_modules.add(module_name) - self._register_module(module) - except Exception as exc: - log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + self._external_modules.add(module_name) + self._register_module(module) + except Exception as exc: + log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr) + finally: + if path_added: + try: + sys.path.remove(plugin_dir_str) + except ValueError: + pass def discover(self) -> None: """Import and register plugins from the package.""" diff --git a/SYS/config.py b/SYS/config.py index 2b79882..9aab825 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -26,12 +26,16 @@ SCRIPT_DIR = Path(__file__).resolve().parent _SAVE_LOCK_DIRNAME = ".medios_save_lock" _SAVE_LOCK_TIMEOUT = 30.0 # seconds to wait for save lock _SAVE_LOCK_STALE_SECONDS = 3600 # consider lock stale after 1 hour +_SAVE_LOCK_POLL_INTERVAL = 0.1 # seconds between lock acquisition attempts + +_WAL_CHECKPOINT_TIMEOUT = 5.0 # seconds for WAL checkpoint connection _CONFIG_CACHE: Dict[str, Any] = {} _LAST_SAVED_CONFIG: Dict[str, Any] = {} _CONFIG_SUMMARY_PENDING = False _CONFIG_SAVE_MAX_RETRIES = 5 _CONFIG_SAVE_RETRY_DELAY = 0.15 +_CONFIG_SAVE_VERIFY_RETRIES = 3 _CONFIG_MISSING = object() _PATH_ALIAS_TOKEN_RE = re.compile(r"^\$(?:\((?P[^)]+)\)|(?P[A-Za-z0-9_.-]+))$") @@ -890,7 +894,7 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT): logger.exception("Failed to inspect save lock directory %s: %s", lock_dir, exc) if time.time() - start > timeout: raise ConfigSaveConflict("Save lock busy; could not acquire in time") - time.sleep(0.1) + time.sleep(_SAVE_LOCK_POLL_INTERVAL) def _release_save_lock(lock_dir: Path) -> None: @@ -1030,10 +1034,10 @@ def save_config(config: Dict[str, Any]) -> int: # we don't contend with our main connection lock or active transactions. try: try: - with sqlite3.connect(str(db.db_path), timeout=5.0) as _con: + with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con: _con.execute("PRAGMA wal_checkpoint(TRUNCATE)") except Exception: - with sqlite3.connect(str(db.db_path), timeout=5.0) as _con: + with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con: _con.execute("PRAGMA wal_checkpoint") except Exception as exc: log(f"Warning: WAL checkpoint failed: {exc}") @@ -1084,7 +1088,7 @@ def save(config: Dict[str, Any]) -> int: return save_config(config) -def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: float = 0.15) -> int: +def save_config_and_verify(config: Dict[str, Any], retries: int = _CONFIG_SAVE_VERIFY_RETRIES, delay: float = _CONFIG_SAVE_RETRY_DELAY) -> int: """Save configuration and verify crucial keys persisted to disk. This helper performs a best-effort verification loop that reloads the diff --git a/SYS/database.py b/SYS/database.py index 29c8140..56dce6e 100644 --- a/SYS/database.py +++ b/SYS/database.py @@ -1,5 +1,6 @@ from __future__ import annotations +import atexit import sqlite3 import json import ast @@ -18,6 +19,8 @@ logger = logging.getLogger(__name__) # DB execute retry settings (for transient 'database is locked' errors) _DB_EXEC_RETRY_MAX = 5 _DB_EXEC_RETRY_BASE_DELAY = 0.05 +_DB_CONNECT_TIMEOUT = 30.0 +_DB_BUSY_TIMEOUT_MS = 30000 # The database is located in the project root (prefer explicit repo hints). def _resolve_root_dir() -> Path: @@ -72,7 +75,7 @@ class Database: self.conn = sqlite3.connect( str(self.db_path), check_same_thread=False, - timeout=30.0 # Increase timeout to 30s to avoid locking issues + timeout=_DB_CONNECT_TIMEOUT ) self.conn.row_factory = sqlite3.Row # Reentrant lock to allow nested DB calls within the same thread (e.g., transaction -> @@ -82,7 +85,7 @@ class Database: # Use WAL mode for better concurrency (allows multiple readers + 1 writer) # Set a busy timeout so SQLite waits for short locks rather than immediately failing try: - self.conn.execute("PRAGMA busy_timeout = 30000") + self.conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}") self.conn.execute("PRAGMA journal_mode=WAL") self.conn.execute("PRAGMA synchronous=NORMAL") except sqlite3.Error: @@ -282,23 +285,27 @@ class Database: # Singleton instance db = Database() -_LOG_QUEUE: Queue[tuple[str, str, str]] = Queue() +_LOG_QUEUE: Queue = Queue() _LOG_THREAD_STARTED = False _LOG_THREAD_LOCK = threading.Lock() _LOG_WRITE_COUNT = 0 _LOG_PRUNE_INTERVAL = 500 # prune every N successful writes _LOG_MAX_AGE_DAYS = 7 # keep logs for this many days +_LOG_WRITE_RETRY_MAX = 3 +_LOG_WRITE_RETRY_BASE_DELAY = 0.05 +_LOG_SENTINEL = object() +_LOG_THREAD_REF: Optional[threading.Thread] = None def _ensure_log_db_schema() -> None: try: conn = sqlite3.connect( str(LOG_DB_PATH), - timeout=30.0, + timeout=_DB_CONNECT_TIMEOUT, check_same_thread=False, ) try: - conn.execute("PRAGMA busy_timeout = 30000") + conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}") conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA synchronous=NORMAL") conn.execute( @@ -328,15 +335,20 @@ def _log_worker_loop() -> None: """ global _LOG_WRITE_COUNT while True: - level, module, message = _LOG_QUEUE.get() + item = _LOG_QUEUE.get() + if item is _LOG_SENTINEL: + break + level, module, message = item try: attempts = 0 written = False - while attempts < 3 and not written: + while attempts < _LOG_WRITE_RETRY_MAX and not written: + conn = None + cur = None try: - conn = sqlite3.connect(str(LOG_DB_PATH), timeout=30.0) + conn = sqlite3.connect(str(LOG_DB_PATH), timeout=_DB_CONNECT_TIMEOUT) try: - conn.execute("PRAGMA busy_timeout = 30000") + conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}") conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA synchronous=NORMAL") except sqlite3.Error: @@ -352,13 +364,11 @@ def _log_worker_loop() -> None: conn.commit() except Exception: pass - cur.close() - conn.close() written = True except sqlite3.OperationalError as exc: attempts += 1 if 'locked' in str(exc).lower(): - time.sleep(0.05 * attempts) + time.sleep(_LOG_WRITE_RETRY_BASE_DELAY * attempts) continue # Non-lock operational errors: abort attempts log(f"Warning: Failed to write log entry (operational): {exc}") @@ -366,6 +376,17 @@ def _log_worker_loop() -> None: except Exception as exc: log(f"Warning: Failed to write log entry: {exc}") break + finally: + if cur is not None: + try: + cur.close() + except Exception: + pass + if conn is not None: + try: + conn.close() + except Exception: + pass if not written: # Fallback to a file-based log so we never lose the message silently try: @@ -399,7 +420,7 @@ def _log_worker_loop() -> None: def _ensure_log_thread() -> None: - global _LOG_THREAD_STARTED + global _LOG_THREAD_STARTED, _LOG_THREAD_REF if _LOG_THREAD_STARTED: return with _LOG_THREAD_LOCK: @@ -411,8 +432,24 @@ def _ensure_log_thread() -> None: daemon=True ) thread.start() + _LOG_THREAD_REF = thread _LOG_THREAD_STARTED = True + +def _shutdown_log_thread() -> None: + global _LOG_THREAD_REF + _LOG_QUEUE.put(_LOG_SENTINEL) + thread = _LOG_THREAD_REF + if thread is not None: + try: + thread.join(timeout=3.0) + except Exception: + pass + _LOG_THREAD_REF = None + + +atexit.register(_shutdown_log_thread) + def get_db() -> Database: return db @@ -533,7 +570,11 @@ def get_config_all() -> Dict[str, Any]: # Worker Management Methods for medios.db -def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection: +_WORKER_DB_DEFAULT_TIMEOUT = 0.75 +_WORKER_DB_DEFAULT_RETRIES = 1 +_WORKER_DB_RETRY_BASE_DELAY = 0.05 + +def _worker_db_connect(timeout: float = _WORKER_DB_DEFAULT_TIMEOUT) -> sqlite3.Connection: conn = sqlite3.connect( str(DB_PATH), timeout=timeout, @@ -555,8 +596,8 @@ def _worker_db_execute( params: tuple = (), *, fetch: Optional[str] = None, - timeout: float = 0.75, - retries: int = 1, + timeout: float = _WORKER_DB_DEFAULT_TIMEOUT, + retries: int = _WORKER_DB_DEFAULT_RETRIES, ) -> Any: attempts = 0 while True: @@ -580,7 +621,7 @@ def _worker_db_execute( msg = str(exc).lower() if "locked" in msg and attempts < retries: attempts += 1 - time.sleep(0.05 * attempts) + time.sleep(_WORKER_DB_RETRY_BASE_DELAY * attempts) continue raise finally: diff --git a/SYS/file_server.py b/SYS/file_server.py index 40914d0..838ec10 100644 --- a/SYS/file_server.py +++ b/SYS/file_server.py @@ -1,9 +1,11 @@ """Simple HTTP file server for serving files in web mode.""" +import atexit +import os import threading import socket import logging -from http.server import HTTPServer, SimpleHTTPRequestHandler +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler from pathlib import Path from typing import Optional import mimetypes @@ -11,9 +13,12 @@ import urllib.parse logger = logging.getLogger(__name__) +_DNS_FALLBACK_HOST = os.environ.get("MM_DNS_SERVER", "8.8.8.8") +_DNS_FALLBACK_PORT = 80 + # Global server instance -_file_server: Optional[HTTPServer] = None -_server_thread: Optional[threading.Thread] = None +_file_server: Optional[ThreadingHTTPServer] = None +_file_server_thread: Optional[threading.Thread] = None _server_port: int = 8001 @@ -84,7 +89,7 @@ def get_local_ip() -> Optional[str]: try: # Connect to a remote server to determine local IP s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("8.8.8.8", 80)) + s.connect((_DNS_FALLBACK_HOST, _DNS_FALLBACK_PORT)) ip = s.getsockname()[0] s.close() return ip @@ -116,7 +121,7 @@ def start_file_server(port: int = 8001) -> Optional[str]: # Create server server_address = ("", port) - _file_server = HTTPServer(server_address, FileServerHandler) + _file_server = ThreadingHTTPServer(server_address, FileServerHandler) # Start in daemon thread _server_thread = threading.Thread( @@ -125,6 +130,8 @@ def start_file_server(port: int = 8001) -> Optional[str]: ) _server_thread.start() + atexit.register(stop_file_server) + logger.info(f"File server started on port {port}") # Get local IP diff --git a/SYS/metadata.py b/SYS/metadata.py index b43d1ab..bf8f96e 100644 --- a/SYS/metadata.py +++ b/SYS/metadata.py @@ -523,8 +523,8 @@ def _read_sidecar_metadata( lower = line.lower() if lower.startswith("hash:"): hash_value = line.split(":", 1)[1].strip() if ":" in line else "" - elif lower.startswith("url:") or lower.startswith("url:"): - # Parse url (handle legacy 'url:' format) + elif lower.startswith("url:"): + # Parse url url_part = line.split(":", 1)[1].strip() if ":" in line else "" if url_part: for url_segment in url_part.split(","): @@ -638,44 +638,7 @@ def write_tags( return except Exception as e: debug(f"Failed to add tags to database: {e}", file=sys.stderr) - # Fall through to sidecar creation as fallback - - # Create sidecar path - try: - sidecar = media_path.parent / (media_path.name + ".tag") - except Exception: - sidecar = media_path.with_name(media_path.name + ".tag") - - # Handle edge case: empty/invalid base name - try: - if not sidecar.stem or sidecar.name in {".tag", - "-.tag", - "_.tag"}: - fallback_base = ( - media_path.stem - or _sanitize_title_for_filename(extract_title(tag_list) or "") - or "untitled" - ) - sidecar = media_path.parent / f"{fallback_base}.tag" - except Exception: - logger.exception("Failed to determine fallback .tag sidecar base for %s", media_path) - - # Write via consolidated function - try: - lines: List[str] = [] - lines.extend(str(tag).strip().lower() for tag in tag_list if str(tag).strip()) - - if lines: - sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8") - if emit_debug: - debug(f"Tags: {sidecar}") - else: - try: - sidecar.unlink() - except FileNotFoundError: - pass - except OSError as exc: - debug(f"Failed to write tag sidecar {sidecar}: {exc}", file=sys.stderr) + return def write_metadata( @@ -729,45 +692,7 @@ def write_metadata( return except Exception as e: debug(f"Failed to add metadata to database: {e}", file=sys.stderr) - # Fall through to sidecar creation as fallback - - # Create sidecar path - try: - sidecar = media_path.parent / (media_path.name + ".metadata") - except Exception: - sidecar = media_path.with_name(media_path.name + ".metadata") - - try: - lines = [] - - # Add hash if available - if hash_value: - lines.append(f"hash:{hash_value}") - - # Add known url - for url in url_list: - if str(url).strip(): - clean = str(url).strip() - lines.append(f"url:{clean}") - - # Add relationships - for rel in rel_list: - if str(rel).strip(): - lines.append(f"relationship:{str(rel).strip()}") - - # Write metadata file - if lines: - sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8") - if emit_debug: - debug(f"Wrote metadata to {sidecar}") - else: - # Remove if no content - try: - sidecar.unlink() - except FileNotFoundError: - pass - except OSError as exc: - debug(f"Failed to write metadata sidecar {sidecar}: {exc}", file=sys.stderr) + return def extract_title(tags: Iterable[str]) -> Optional[str]: @@ -2864,17 +2789,7 @@ def build_ffmpeg_command( "192k", ]) cmd.extend(["-f", "opus"]) - elif fmt == "audio": - # Legacy format name for mp3 - cmd.extend([ - "-vn", - "-c:a", - "libmp3lame", - "-b:a", - "192k", - ]) - cmd.extend(["-f", "mp3"]) - elif fmt != "copy": + if fmt not in ("mp4", "webm", "mp3", "flac", "wav", "aac", "m4a", "ogg", "opus", "copy"): raise ValueError(f"Unsupported format: {fmt}") cmd.append(str(output_path)) diff --git a/SYS/pipeline.py b/SYS/pipeline.py index e549516..f1eaff4 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1,3317 +1,15 @@ """ -Pipeline execution context and state management for cmdlet. +Re-export module for backward compatibility. + +@deprecated: Prefer direct imports from SYS.pipeline_state or SYS.pipeline_executor. + - from SYS.pipeline_state import PipelineState, get_pipeline_state, ... + - from SYS.pipeline_executor import PipelineExecutor + +This module re-exports all names from both submodules so existing code +that imports from SYS.pipeline continues to work unchanged. """ -from __future__ import annotations +from SYS.pipeline_state import __all__ as _state_all +from SYS.pipeline_state import * # noqa: F401, F403 +from SYS.pipeline_executor import PipelineExecutor # noqa: F401 -import sys -import time -from contextlib import contextmanager -from dataclasses import dataclass, field -from contextvars import ContextVar -from typing import Any, Dict, List, Optional, Sequence, Callable -from SYS.models import PipelineStageContext -from SYS.logger import log, debug, debug_panel, is_debug_enabled -import logging -logger = logging.getLogger(__name__) -# SYS.worker deferred: ffmpeg+attr+rich (~260ms) loaded lazily on first pipeline run. -_worker_mod: Any = None -# SYS.cli_parsing deferred: prompt_toolkit (~300ms) loaded lazily on first selection. -_cli_parsing_mod: Any = None - - -def _worker(): - global _worker_mod - if _worker_mod is None: - import SYS.worker as _m - _worker_mod = _m - return _worker_mod - - -def _cli_parsing(): - global _cli_parsing_mod - if _cli_parsing_mod is None: - import SYS.cli_parsing as _m - _cli_parsing_mod = _m - return _cli_parsing_mod - - -# SYS.rich_display deferred: rich (~100ms) loaded lazily on first console output. -# SYS.background_notifier deferred: rich/attr/ffmpeg loaded lazily on first notifier use. -# SYS.result_table deferred: textual (~140ms) loaded lazily on first Table use. -_result_table_mod: Any = None - - -def _result_table(): - global _result_table_mod - if _result_table_mod is None: - from SYS.result_table import Table as _T - _result_table_mod = _T - return _result_table_mod - -import re -from datetime import datetime -from SYS.cmdlet_catalog import import_cmd_module - -HELP_EXAMPLE_SOURCE_COMMANDS = { - ".help-example", - "help-example", -} - - -def _emit_selection_debug_panel( - *, - selection_token: Any, - selection_indices: Sequence[int], - item_count: int, - filtered_count: int, - stage_table_present: bool, - display_table_present: bool, - stage_is_last: bool, - row_action: Optional[Sequence[Any]] = None, - downstream_stages: Optional[Sequence[Sequence[Any]]] = None, - mode: Optional[str] = None, -) -> None: - if not is_debug_enabled(): - return - - try: - rows: List[tuple[str, Any]] = [ - ("selection", str(selection_token or "")), - ("indices", [int(idx) + 1 for idx in (selection_indices or [])]), - ("items", int(item_count)), - ("filtered", int(filtered_count)), - ("stage_table", bool(stage_table_present)), - ("display_table", bool(display_table_present)), - ("stage_is_last", bool(stage_is_last)), - ("downstream_stages", len(list(downstream_stages or []))), - ] - if mode: - rows.insert(1, ("mode", str(mode))) - if row_action: - rows.append(("row_action", " ".join(str(part) for part in row_action if part is not None))) - - debug_panel( - f"Selection replay {selection_token}", - rows, - border_style="magenta", - ) - except Exception: - pass - - -def set_live_progress(progress_ui: Any) -> None: - """Register the current Live progress UI so cmdlets can suspend it during prompts.""" - state = _get_pipeline_state() - state.live_progress = progress_ui - - -def get_live_progress() -> Any: - state = _get_pipeline_state() - return state.live_progress - - -def set_progress_event_callback(callback: Any) -> None: - state = _get_pipeline_state() - state.progress_event_callback = callback - - -def get_progress_event_callback() -> Any: - state = _get_pipeline_state() - return state.progress_event_callback - - -@contextmanager -def suspend_live_progress(): - """Temporarily pause Live progress rendering. - - This avoids Rich Live cursor control interfering with interactive tables/prompts - emitted by cmdlets during preflight (e.g. URL-duplicate confirmation). - """ - ui = get_live_progress() - paused = False - try: - if ui is not None and hasattr(ui, "pause"): - try: - ui.pause() - paused = True - except Exception as exc: - logger.exception("Failed to pause live progress UI: %s", exc) - paused = False - yield - finally: - # If a stage requested the pipeline stop (e.g. user declined a preflight prompt), - # do not resume Live rendering. - if get_pipeline_stop() is not None: - return - if paused and ui is not None and hasattr(ui, "resume"): - try: - ui.resume() - except Exception: - logger.exception("Failed to resume live progress UI after suspend") - - -def _is_selectable_table(table: Any) -> bool: - """Return True when a table can be used for @ selection.""" - # Avoid relying on truthiness for selectability. - # `ResultTable` can be falsey when it has 0 rows, but `@` selection/filtering - # should still be allowed when the backing `last_result_items` exist. - return table is not None and not getattr(table, "no_choice", False) - - -# Pipeline state container (prototype) -@dataclass -class PipelineState: - current_context: Optional[PipelineStageContext] = None - last_search_query: Optional[str] = None - pipeline_refreshed: bool = False - last_items: List[Any] = field(default_factory=list) - last_result_table: Optional[Any] = None - last_result_items: List[Any] = field(default_factory=list) - last_result_subject: Optional[Any] = None - result_table_history: List[tuple[Optional[Any], List[Any], Optional[Any]]] = field(default_factory=list) - result_table_forward: List[tuple[Optional[Any], List[Any], Optional[Any]]] = field(default_factory=list) - current_stage_table: Optional[Any] = None - display_items: List[Any] = field(default_factory=list) - display_table: Optional[Any] = None - display_subject: Optional[Any] = None - last_selection: List[int] = field(default_factory=list) - pipeline_command_text: str = "" - current_cmdlet_name: str = "" - current_stage_text: str = "" - pipeline_values: Dict[str, Any] = field(default_factory=dict) - pending_pipeline_tail: List[List[str]] = field(default_factory=list) - pending_pipeline_source: Optional[str] = None - ui_library_refresh_callback: Optional[Any] = None - pipeline_stop: Optional[Dict[str, Any]] = None - live_progress: Any = None - last_execution_result: Dict[str, Any] = field(default_factory=dict) - progress_event_callback: Any = None - - def reset(self) -> None: - self.current_context = None - self.last_search_query = None - self.pipeline_refreshed = False - self.last_items = [] - self.last_result_table = None - self.last_result_items = [] - self.last_result_subject = None - self.result_table_history = [] - self.result_table_forward = [] - self.current_stage_table = None - self.display_items = [] - self.display_table = None - self.display_subject = None - self.last_selection = [] - self.pipeline_command_text = "" - self.current_cmdlet_name = "" - self.current_stage_text = "" - self.pipeline_values = {} - self.pending_pipeline_tail = [] - self.pending_pipeline_source = None - self.ui_library_refresh_callback = None - self.pipeline_stop = None - self.live_progress = None - self.last_execution_result = {} - self.progress_event_callback = None - - -# ContextVar for per-run state (prototype) -_CTX_STATE: ContextVar[Optional[PipelineState]] = ContextVar("_pipeline_state", default=None) -_GLOBAL_STATE: PipelineState = PipelineState() - - -def _get_pipeline_state() -> PipelineState: - """Return the PipelineState for the current context or the global fallback.""" - state = _CTX_STATE.get() - return state if state is not None else _GLOBAL_STATE - - -@contextmanager -def new_pipeline_state(): - """Context manager to use a fresh PipelineState for a run.""" - token = _CTX_STATE.set(PipelineState()) - try: - yield _CTX_STATE.get() - finally: - _CTX_STATE.reset(token) - - -# Legacy module-level synchronization removed — module-level pipeline globals are no longer maintained. -# Use `get_pipeline_state()` to access or mutate the per-run PipelineState. - - - -# Public accessors for pipeline state (for external callers that need to inspect -# or mutate the PipelineState directly). These provide stable, non-underscored -# entrypoints so other modules don't rely on implementation-internal names. -def get_pipeline_state() -> PipelineState: - """Return the active PipelineState for the current context or the global fallback.""" - return _get_pipeline_state() - - - - -# No module-level pipeline runtime variables; per-run pipeline state is stored in PipelineState (use `get_pipeline_state()`). -MAX_RESULT_TABLE_HISTORY = 20 -PIPELINE_MISSING = object() - - -def request_pipeline_stop(*, reason: str = "", exit_code: int = 0) -> None: - """Request that the pipeline runner stop gracefully after the current stage.""" - state = _get_pipeline_state() - state.pipeline_stop = { - "reason": str(reason or "").strip(), - "exit_code": int(exit_code) - } - - -def get_pipeline_stop() -> Optional[Dict[str, Any]]: - state = _get_pipeline_state() - return state.pipeline_stop - - -def clear_pipeline_stop() -> None: - state = _get_pipeline_state() - state.pipeline_stop = None - - -# ============================================================================ -# PUBLIC API -# ============================================================================ - - -def set_stage_context(context: Optional[PipelineStageContext]) -> None: - """Set the current pipeline stage context.""" - state = _get_pipeline_state() - state.current_context = context - - -def get_stage_context() -> Optional[PipelineStageContext]: - """Get the current pipeline stage context.""" - state = _get_pipeline_state() - return state.current_context - - -def emit(obj: Any) -> None: - """ - Emit an object to the current pipeline stage output. - """ - ctx = _get_pipeline_state().current_context - if ctx is not None: - ctx.emit(obj) - - -def emit_list(objects: List[Any]) -> None: - """ - Emit a list of objects to the next pipeline stage. - """ - ctx = _get_pipeline_state().current_context - if ctx is not None: - ctx.emit(objects) - - -def print_if_visible(*args: Any, file=None, **kwargs: Any) -> None: - """ - Print only if this is not a quiet mid-pipeline stage. - """ - try: - # Print if: not in a pipeline OR this is the last stage - ctx = _get_pipeline_state().current_context - should_print = (ctx is None) or (ctx and ctx.is_last_stage) - - # Always print to stderr regardless - if file is not None: - should_print = True - - if should_print: - log(*args, **kwargs) if file is None else log(*args, file=file, **kwargs) - except Exception: - logger.exception("Error in print_if_visible") - - -def store_value(key: str, value: Any) -> None: - """ - Store a value to pass to later pipeline stages. - """ - if not isinstance(key, str): - return - text = key.strip().lower() - if not text: - return - try: - state = _get_pipeline_state() - state.pipeline_values[text] = value - except Exception: - logger.exception("Failed to store pipeline value '%s'", key) - - -def load_value(key: str, default: Any = None) -> Any: - """ - Retrieve a value stored by an earlier pipeline stage. - """ - if not isinstance(key, str): - return default - text = key.strip() - if not text: - return default - parts = [segment.strip() for segment in text.split(".") if segment.strip()] - if not parts: - return default - root_key = parts[0].lower() - state = _get_pipeline_state() - container = state.pipeline_values.get(root_key, PIPELINE_MISSING) - if container is PIPELINE_MISSING: - return default - if len(parts) == 1: - return container - - current: Any = container - for fragment in parts[1:]: - if isinstance(current, dict): - fragment_lower = fragment.lower() - if fragment in current: - current = current[fragment] - continue - match = PIPELINE_MISSING - for key_name, value in current.items(): - if isinstance(key_name, str) and key_name.lower() == fragment_lower: - match = value - break - if match is PIPELINE_MISSING: - return default - current = match - continue - if isinstance(current, (list, tuple)): - if fragment.isdigit(): - try: - idx = int(fragment) - except ValueError: - return default - if 0 <= idx < len(current): - current = current[idx] - continue - return default - if hasattr(current, fragment): - try: - current = getattr(current, fragment) - continue - except Exception: - return default - return default - return current - - -def set_last_execution_result( - *, - status: str, - error: str = "", - command_text: str = "", -) -> None: - state = _get_pipeline_state() - text_status = str(status or "").strip().lower() or "unknown" - state.last_execution_result = { - "status": text_status, - "success": text_status == "completed", - "error": str(error or "").strip(), - "command_text": str(command_text or "").strip(), - "finished_at": time.time(), - } - - -def get_last_execution_result() -> Dict[str, Any]: - state = _get_pipeline_state() - payload = state.last_execution_result - return dict(payload) if isinstance(payload, dict) else {} - - -def set_pending_pipeline_tail( - stages: Optional[Sequence[Sequence[str]]], - source_command: Optional[str] = None -) -> None: - """ - Store the remaining pipeline stages when execution pauses for @N selection. - """ - state = _get_pipeline_state() - try: - pending: List[List[str]] = [] - for stage in stages or []: - if isinstance(stage, (list, tuple)): - pending.append([str(token) for token in stage]) - state.pending_pipeline_tail = pending - clean_source = (source_command or "").strip() - state.pending_pipeline_source = clean_source if clean_source else None - except Exception: - # Keep existing pending tail on failure - logger.exception("Failed to set pending pipeline tail; keeping existing pending tail") - - -def get_pending_pipeline_tail() -> List[List[str]]: - """Get a copy of the pending pipeline tail (stages queued after selection).""" - state = _get_pipeline_state() - return [list(stage) for stage in state.pending_pipeline_tail] - - -def get_pending_pipeline_source() -> Optional[str]: - """Get the source command associated with the pending pipeline tail.""" - state = _get_pipeline_state() - return state.pending_pipeline_source - - -def clear_pending_pipeline_tail() -> None: - """Clear any stored pending pipeline tail.""" - state = _get_pipeline_state() - state.pending_pipeline_tail = [] - state.pending_pipeline_source = None - - -def reset() -> None: - """Reset all pipeline state. Called between pipeline executions.""" - state = _get_pipeline_state() - state.reset() - - -def get_emitted_items() -> List[Any]: - """ - Get a copy of all items emitted by the current pipeline stage. - """ - state = _get_pipeline_state() - ctx = state.current_context - if ctx is not None: - return list(ctx.emits) - return [] - - -def clear_emits() -> None: - """Clear the emitted items list (called between stages).""" - state = _get_pipeline_state() - ctx = state.current_context - if ctx is not None: - ctx.emits.clear() - - -def set_last_selection(indices: Sequence[int]) -> None: - """Record the indices selected via @ syntax for the next cmdlet. - - Args: - indices: Iterable of 0-based indices captured from the REPL parser - """ - state = _get_pipeline_state() - state.last_selection = list(indices or []) - - -def get_last_selection() -> List[int]: - """Return the indices selected via @ syntax for the current invocation.""" - state = _get_pipeline_state() - return list(state.last_selection) - - -def clear_last_selection() -> None: - """Clear the cached selection indices after a cmdlet finishes.""" - state = _get_pipeline_state() - state.last_selection = [] - - -def set_current_command_text(command_text: Optional[str]) -> None: - """Record the raw pipeline/command text for downstream consumers.""" - state = _get_pipeline_state() - state.pipeline_command_text = (command_text or "").strip() - - -def get_current_command_text(default: str = "") -> str: - """Return the last recorded command/pipeline text.""" - state = _get_pipeline_state() - text = state.pipeline_command_text.strip() - return text if text else default - - -def clear_current_command_text() -> None: - """Clear the cached command text after execution completes.""" - state = _get_pipeline_state() - state.pipeline_command_text = "" - - -def split_pipeline_text(pipeline_text: str) -> List[str]: - """Split a pipeline string on unquoted '|' characters. - - Preserves original quoting/spacing within each returned stage segment. - """ - text = str(pipeline_text or "") - if not text: - return [] - - stages: List[str] = [] - buf: List[str] = [] - quote: Optional[str] = None - escape = False - - for ch in text: - if escape: - buf.append(ch) - escape = False - continue - - if ch == "\\" and quote is not None: - buf.append(ch) - escape = True - continue - - if ch in ('"', "'"): - if quote is None: - quote = ch - elif quote == ch: - quote = None - buf.append(ch) - continue - - if ch == "|" and quote is None: - stages.append("".join(buf).strip()) - buf = [] - continue - - buf.append(ch) - - tail = "".join(buf).strip() - if tail: - stages.append(tail) - return [s for s in stages if s] - - -def get_current_command_stages() -> List[str]: - """Return the raw stage segments for the current command text.""" - return split_pipeline_text(get_current_command_text("")) - - -def set_current_stage_text(stage_text: Optional[str]) -> None: - """Record the raw stage text currently being executed.""" - state = _get_pipeline_state() - state.current_stage_text = str(stage_text or "").strip() - - -def get_current_stage_text(default: str = "") -> str: - """Return the raw stage text currently being executed.""" - state = _get_pipeline_state() - text = state.current_stage_text.strip() - return text if text else default - - -def clear_current_stage_text() -> None: - """Clear the cached stage text after a stage completes.""" - state = _get_pipeline_state() - state.current_stage_text = "" - - -def set_current_cmdlet_name(cmdlet_name: Optional[str]) -> None: - """Record the currently executing cmdlet name (stage-local).""" - state = _get_pipeline_state() - state.current_cmdlet_name = str(cmdlet_name or "").strip() - - -def get_current_cmdlet_name(default: str = "") -> str: - """Return the currently executing cmdlet name (stage-local).""" - state = _get_pipeline_state() - text = state.current_cmdlet_name.strip() - return text if text else default - - -def clear_current_cmdlet_name() -> None: - """Clear the cached cmdlet name after a stage completes.""" - state = _get_pipeline_state() - state.current_cmdlet_name = "" - - -def set_search_query(query: Optional[str]) -> None: - """Set the last search query for refresh purposes.""" - state = _get_pipeline_state() - state.last_search_query = query - - -def get_search_query() -> Optional[str]: - """Get the last search query.""" - state = _get_pipeline_state() - return state.last_search_query - - -def set_pipeline_refreshed(refreshed: bool) -> None: - """Track whether the pipeline already refreshed results.""" - state = _get_pipeline_state() - state.pipeline_refreshed = refreshed - - -def was_pipeline_refreshed() -> bool: - """Check if the pipeline already refreshed results.""" - state = _get_pipeline_state() - return state.pipeline_refreshed - - -def set_last_items(items: list) -> None: - """Cache the last pipeline outputs.""" - state = _get_pipeline_state() - state.last_items = list(items) if items else [] - - -def get_last_items() -> List[Any]: - """Get the last pipeline outputs.""" - state = _get_pipeline_state() - return list(state.last_items) - - -def set_ui_library_refresh_callback(callback: Any) -> None: - """ - Set a callback to be called when library content is updated. - """ - state = _get_pipeline_state() - state.ui_library_refresh_callback = callback - - -def get_ui_library_refresh_callback() -> Optional[Any]: - """Get the current library refresh callback.""" - state = _get_pipeline_state() - return state.ui_library_refresh_callback - - -def trigger_ui_library_refresh(library_filter: str = "local") -> None: - """Trigger a library refresh in the UI if callback is registered. - - This should be called from cmdlet/funacts after content is added to library. - - Args: - library_filter: Which library to refresh ('local', 'hydrus', etc) - """ - callback = get_ui_library_refresh_callback() - if callback: - try: - callback(library_filter) - except Exception as e: - print( - f"[trigger_ui_library_refresh] Error calling refresh callback: {e}", - file=sys.stderr - ) - - -def set_last_result_table( - result_table: Optional[Any], - items: Optional[List[Any]] = None, - subject: Optional[Any] = None -) -> None: - """Store the last result table and items for @ selection syntax. - - Persists result table and items across command invocations, enabling - subsequent commands to reference and operate on previous results using @N syntax. - - Example: - search-file hash:<...> # Returns table with 3 results - @1 | get-metadata # Gets metadata for result #1 - @2 | add-tag foo # Adds tag to result #2 - - Args: - result_table: Table object with results (can be None to clear) - items: List of item objects corresponding to table rows - subject: Optional context object (first item or full list) - """ - state = _get_pipeline_state() - - # Push current table to history before replacing. - # No .copy() needed: last_result_items is about to be replaced by reference, - # not mutated in place, so the old list reference is safe to keep in history. - if state.last_result_table is not None: - state.result_table_history.append( - ( - state.last_result_table, - state.last_result_items, - state.last_result_subject, - ) - ) - # Keep history size limited - if len(state.result_table_history) > MAX_RESULT_TABLE_HISTORY: - state.result_table_history.pop(0) - - # Set new current table and clear any display items/table - state.display_items = [] - state.display_table = None - state.display_subject = None - state.last_result_table = result_table - state.last_result_items = items or [] - state.last_result_subject = subject - - # Sort table by Title/Name column alphabetically if available - if ( - result_table is not None - and hasattr(result_table, "sort_by_title") - and not getattr(result_table, "preserve_order", False) - ): - try: - result_table.sort_by_title() - # Re-order items list to match the sorted table - if state.last_result_items and hasattr(result_table, "rows"): - sorted_items: List[Any] = [] - for row in result_table.rows: - src_idx = getattr(row, "source_index", None) - if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items): - sorted_items.append(state.last_result_items[src_idx]) - # Only reassign when the table actually contains rows and the reordering - # produced a complete mapping. Avoid clearing items when the table has no rows. - if result_table.rows and len(sorted_items) == len(result_table.rows): - state.last_result_items = sorted_items - except Exception: - logger.exception("Failed to sort result_table and reorder items") - - -def set_last_result_table_overlay( - result_table: Optional[Any], - items: Optional[List[Any]] = None, - subject: Optional[Any] = None -) -> None: - """Store a display table and items WITHOUT affecting history stack. - - Used by action cmdlets (get-metadata, get-tag, get-url) to display detail - panels or filtered results without disrupting the primary search-result history. - - Difference from set_last_result_table(): - - Overlay tables are transient (in-process memory only) - - Don't persist across command invocations - - Used for "live" displays that shouldn't be part of @N selection - - Args: - result_table: Table object with transient results - items: List of item objects (not persisted) - subject: Optional context object - """ - state = _get_pipeline_state() - state.display_table = result_table - state.display_items = items or [] - state.display_subject = subject - -def set_last_result_items_only(items: Optional[List[Any]]) -> None: - """ - Store items for @N selection WITHOUT affecting history or saved search data. - """ - state = _get_pipeline_state() - - # Store items for immediate @N selection, but DON'T modify last_result_items - # This ensures history contains original search data, not display transformations - state.display_items = items or [] - # Clear display table since we're setting items only (CLI will generate table if needed) - state.display_table = None - state.display_subject = None - - -def restore_previous_result_table() -> bool: - """ - Restore the previous result table from history (for @.. navigation). - """ - state = _get_pipeline_state() - - # If we have an active overlay (display items/table), clear it to "go back" to the underlying table - if state.display_items or state.display_table or state.display_subject is not None: - state.display_items = [] - state.display_table = None - state.display_subject = None - # If an underlying table exists, we're done. - # Otherwise, fall through to history restore so @.. actually returns to the last table. - if state.last_result_table is not None: - # Ensure subsequent @N selection uses the table the user sees. - state.current_stage_table = state.last_result_table - return True - if not state.result_table_history: - state.current_stage_table = state.last_result_table - return True - - if not state.result_table_history: - return False - - # Save current state to forward stack before popping - state.result_table_forward.append( - (state.last_result_table, state.last_result_items, state.last_result_subject) - ) - - # Pop from history and restore - prev = state.result_table_history.pop() - if isinstance(prev, tuple) and len(prev) >= 3: - state.last_result_table, state.last_result_items, state.last_result_subject = prev[0], prev[1], prev[2] - elif isinstance(prev, tuple) and len(prev) == 2: - state.last_result_table, state.last_result_items = prev - state.last_result_subject = None - else: - state.last_result_table, state.last_result_items, state.last_result_subject = None, [], None - - # Clear display items so get_last_result_items() falls back to restored items - state.display_items = [] - state.display_table = None - state.display_subject = None - - # Sync current stage table to the restored view so provider selectors run - # against the correct table type. - state.current_stage_table = state.last_result_table - - try: - debug_table_state("restore_previous_result_table") - except Exception: - logger.exception("Failed to debug_table_state during restore_previous_result_table") - - return True - - -def restore_next_result_table() -> bool: - """ - Restore the next result table from forward history (for @,, navigation). - """ - state = _get_pipeline_state() - - # If we have an active overlay (display items/table), clear it to "go forward" to the underlying table - if state.display_items or state.display_table or state.display_subject is not None: - state.display_items = [] - state.display_table = None - state.display_subject = None - # If an underlying table exists, we're done. - # Otherwise, fall through to forward restore when available. - if state.last_result_table is not None: - # Ensure subsequent @N selection uses the table the user sees. - state.current_stage_table = state.last_result_table - return True - if not state.result_table_forward: - state.current_stage_table = state.last_result_table - return True - - if not state.result_table_forward: - return False - - # Save current state to history stack before popping forward - state.result_table_history.append( - (state.last_result_table, state.last_result_items, state.last_result_subject) - ) - - # Pop from forward stack and restore - next_state = state.result_table_forward.pop() - if isinstance(next_state, tuple) and len(next_state) >= 3: - state.last_result_table, state.last_result_items, state.last_result_subject = ( - next_state[0], next_state[1], next_state[2] - ) - elif isinstance(next_state, tuple) and len(next_state) == 2: - state.last_result_table, state.last_result_items = next_state - state.last_result_subject = None - else: - state.last_result_table, state.last_result_items, state.last_result_subject = None, [], None - - # Clear display items so get_last_result_items() falls back to restored items - state.display_items = [] - state.display_table = None - state.display_subject = None - - # Sync current stage table to the restored view so provider selectors run - # against the correct table type. - state.current_stage_table = state.last_result_table - - try: - debug_table_state("restore_next_result_table") - except Exception: - logger.exception("Failed to debug_table_state during restore_next_result_table") - - return True - - -def get_display_table() -> Optional[Any]: - """ - Get the current display overlay table. - """ - state = _get_pipeline_state() - return state.display_table - - -def get_last_result_subject() -> Optional[Any]: - """ - Get the subject associated with the current result table or overlay. - """ - state = _get_pipeline_state() - if state.display_subject is not None: - return state.display_subject - return state.last_result_subject - - -def get_last_result_table() -> Optional[Any]: - """Get the current last result table. - - Returns: - The ResultTable object, or None if no table is set - """ - state = _get_pipeline_state() - return state.last_result_table - - -def get_last_result_items() -> List[Any]: - """Get the items available for @N selection in current pipeline context. - - Returns items in priority order: - 1. Display items (from get-tag, get-metadata, etc.) if display table is selectable - 2. Last result items (from search-file, etc.) if last result table is selectable - 3. Empty list if no selectable tables available - - Used to resolve @1, @2, etc. in commands. - - Returns: - List of items that can be selected via @N syntax - """ - state = _get_pipeline_state() - # Prioritize items from display commands (get-tag, delete-tag, etc.) - # These are available for immediate @N selection - if state.display_items: - if state.display_table is not None and not _is_selectable_table(state.display_table): - return [] - return state.display_items - # Fall back to items from last search/selectable command - if state.last_result_table is None: - return state.last_result_items - if _is_selectable_table(state.last_result_table): - return state.last_result_items - return [] - - -def debug_table_state(label: str = "") -> None: - """Dump pipeline table and item-buffer state (debug-only). - - Useful for diagnosing cases where `@N` selection appears to act on a different - table than the one currently displayed. - """ - - if not is_debug_enabled(): - return - - state = _get_pipeline_state() - - def _tbl(name: str, t: Any) -> None: - if t is None: - debug(f"[table] {name}: None") - return - try: - table_type = getattr(t, "table", None) - except Exception: - table_type = None - try: - title = getattr(t, "title", None) - except Exception: - title = None - try: - src_cmd = getattr(t, "source_command", None) - except Exception: - src_cmd = None - try: - src_args = getattr(t, "source_args", None) - except Exception: - src_args = None - try: - no_choice = bool(getattr(t, "no_choice", False)) - except Exception: - no_choice = False - try: - preserve_order = bool(getattr(t, "preserve_order", False)) - except Exception: - preserve_order = False - try: - row_count = len(getattr(t, "rows", []) or []) - except Exception: - row_count = 0 - try: - meta = ( - t.get_table_metadata() if hasattr(t, "get_table_metadata") else getattr(t, "table_metadata", None) - ) - except Exception: - meta = None - meta_keys = list(meta.keys()) if isinstance(meta, dict) else [] - - debug( - f"[table] {name}: id={id(t)} class={type(t).__name__} title={repr(title)} table={repr(table_type)} rows={row_count} " - f"source={repr(src_cmd)} source_args={repr(src_args)} no_choice={no_choice} preserve_order={preserve_order} meta_keys={meta_keys}" - ) - - if label: - debug(f"[table] state: {label}") - _tbl("display_table", getattr(state, "display_table", None)) - _tbl("current_stage_table", getattr(state, "current_stage_table", None)) - _tbl("last_result_table", getattr(state, "last_result_table", None)) - - try: - debug( - f"[table] buffers: display_items={len(state.display_items or [])} last_result_items={len(state.last_result_items or [])} " - f"history={len(state.result_table_history or [])} forward={len(state.result_table_forward or [])} last_selection={list(state.last_selection or [])}" - ) - except Exception: - logger.exception("Failed to debug_table_state buffers summary") - - -def get_last_selectable_result_items() -> List[Any]: - """Get items from the last *selectable* result table, ignoring display-only items. - - This is useful when a selection stage should target the last visible selectable table - (e.g., a playlist/search table), even if a prior action command emitted items and - populated display_items. - """ - state = _get_pipeline_state() - if state.last_result_table is None: - return list(state.last_result_items) - if _is_selectable_table(state.last_result_table): - return list(state.last_result_items) - return [] - - -def get_last_result_table_source_command() -> Optional[str]: - """Get the source command from the last displayed result table. - - Returns: - Command name (e.g., 'download-file') or None if not set - """ - state = _get_pipeline_state() - table = state.last_result_table - if table is not None and _is_selectable_table(table) and hasattr(table, "source_command"): - return getattr(table, "source_command") - return None - - -def get_last_result_table_source_args() -> List[str]: - """Get the base source arguments from the last displayed result table. - - Returns: - List of arguments (e.g., ['https://example.com']) or empty list - """ - state = _get_pipeline_state() - table = state.last_result_table - if table is not None and _is_selectable_table(table) and hasattr(table, "source_args"): - return getattr(table, "source_args") or [] - return [] - - -def get_last_result_table_row_selection_args(row_index: int) -> Optional[List[str]]: - """Get the selection arguments for a specific row in the last result table. - - Args: - row_index: Index of the row (0-based) - - Returns: - Selection arguments (e.g., ['-item', '3']) or None - """ - state = _get_pipeline_state() - table = state.last_result_table - if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): - rows = table.rows - if 0 <= row_index < len(rows): - row = rows[row_index] - if hasattr(row, "selection_args"): - return getattr(row, "selection_args") - return None - - -def get_last_result_table_row_selection_action(row_index: int) -> Optional[List[str]]: - """Get the expanded stage tokens for a row in the last result table.""" - state = _get_pipeline_state() - table = state.last_result_table - if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): - rows = table.rows - if 0 <= row_index < len(rows): - row = rows[row_index] - if hasattr(row, "selection_action"): - return getattr(row, "selection_action") - return None - -def set_current_stage_table(result_table: Optional[Any]) -> None: - """Store the current pipeline stage table for @N expansion. - - Used by cmdlet that display tabular results (e.g., download-file listing formats) - to make their result table available for @N expansion logic. - - Does NOT push to history - purely for command expansion in the current pipeline. - - Args: - result_table: The ResultTable object (or None to clear) - """ - state = _get_pipeline_state() - state.current_stage_table = result_table - - -def get_current_stage_table() -> Optional[Any]: - """Get the current pipeline stage table (if any).""" - state = _get_pipeline_state() - return state.current_stage_table - - -def get_current_stage_table_source_command() -> Optional[str]: - """Get the source command from the current pipeline stage table. - - Returns: - Command name (e.g., 'download-file') or None - """ - state = _get_pipeline_state() - table = state.current_stage_table - if table is not None and _is_selectable_table(table) and hasattr(table, "source_command"): - return getattr(table, "source_command") - return None - - -def get_current_stage_table_source_args() -> List[str]: - """Get the source arguments from the current pipeline stage table. - - Returns: - List of arguments or empty list - """ - state = _get_pipeline_state() - table = state.current_stage_table - if table is not None and _is_selectable_table(table) and hasattr(table, "source_args"): - return getattr(table, "source_args") or [] - return [] - - -def get_current_stage_table_row_selection_args(row_index: int) -> Optional[List[str]]: - """Get the selection arguments for a row in the current pipeline stage table. - - Args: - row_index: Index of the row (0-based) - - Returns: - Selection arguments or None - """ - state = _get_pipeline_state() - table = state.current_stage_table - if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): - rows = table.rows - if 0 <= row_index < len(rows): - row = rows[row_index] - if hasattr(row, "selection_args"): - return getattr(row, "selection_args") - return None - - -def get_current_stage_table_row_selection_action(row_index: int) -> Optional[List[str]]: - """Get the expanded stage tokens for a row in the current stage table.""" - state = _get_pipeline_state() - table = state.current_stage_table - if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): - rows = table.rows - if 0 <= row_index < len(rows): - row = rows[row_index] - if hasattr(row, "selection_action"): - return getattr(row, "selection_action") - return None - - -def get_current_stage_table_row_source_index(row_index: int) -> Optional[int]: - """Get the original source index for a row in the current stage table. - - Useful when the table has been sorted for display but selections should map - back to the original item order (e.g., playlist or provider order). - """ - state = _get_pipeline_state() - table = state.current_stage_table - if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): - rows = table.rows - if 0 <= row_index < len(rows): - row = rows[row_index] - return getattr(row, "source_index", None) - return None - - -def clear_last_result() -> None: - """Clear the stored last result table and items.""" - state = _get_pipeline_state() - state.last_result_table = None - state.last_result_items = [] - state.last_result_subject = None - - -def _split_pipeline_tokens(tokens: Sequence[str]) -> List[List[str]]: - """Split example tokens into per-stage command sequences using pipe separators.""" - stages: List[List[str]] = [] - current: List[str] = [] - for token in tokens: - if token == "|": - if current: - stages.append(current) - current = [] - continue - current.append(str(token)) - if current: - stages.append(current) - return [stage for stage in stages if stage] - - -class PipelineExecutor: - - def __init__(self, *, config_loader: Optional[Any] = None) -> None: - self._config_loader = config_loader - self._toolbar_output: Optional[Callable[[str], None]] = None - - def _load_config(self) -> Dict[str, Any]: - try: - if self._config_loader is not None: - return self._config_loader.load() - except Exception: - logger.exception("Failed to use config_loader.load(); falling back to SYS.config.load_config") - try: - from SYS.config import load_config - - return load_config() - except Exception: - return {} - - def set_toolbar_output(self, output: Optional[Callable[[str], None]]) -> None: - self._toolbar_output = output - - @staticmethod - def _split_stages(tokens: Sequence[str]) -> List[List[str]]: - stages: List[List[str]] = [] - current: List[str] = [] - for token in tokens: - if token == "|": - if current: - stages.append(current) - current = [] - else: - current.append(token) - if current: - stages.append(current) - return stages - - @staticmethod - def _validate_download_file_relationship_order(stages: List[List[str]]) -> bool: - """Guard against running add-relationship on unstored download-file results. - - Intended UX: - download-file ... | add-file -instance | add-relationship - - Rationale: - download-file outputs items that may not yet have a stable store+hash. - add-relationship is designed to operate in store/hash mode. - """ - - def _norm(name: str) -> str: - return str(name or "").replace("_", "-").strip().lower() - - def _file_action(stage_tokens: List[str]) -> str | None: - if not stage_tokens: - return None - head = _norm(stage_tokens[0]) - if head in {"download-file", "add-file"}: - return head - if head != "file": - return None - args = {_norm(t) for t in stage_tokens[1:]} - if "-download" in args or "--download" in args or "-dl" in args or "--dl" in args: - return "download-file" - if "-add" in args or "--add" in args: - return "add-file" - return None - - names: List[str] = [] - for stage in stages or []: - if not stage: - continue - names.append(_norm(stage[0])) - - dl_idxs = [i for i, stage in enumerate(stages or []) if _file_action(stage or []) == "download-file"] - rel_idxs = [i for i, n in enumerate(names) if n == "add-relationship"] - add_file_idxs = [i for i, stage in enumerate(stages or []) if _file_action(stage or []) == "add-file"] - - if not dl_idxs or not rel_idxs: - return True - - # If download-file is upstream of add-relationship, require an add-file in between. - for rel_i in rel_idxs: - dl_before = [d for d in dl_idxs if d < rel_i] - if not dl_before: - continue - dl_i = max(dl_before) - if not any(dl_i < a < rel_i for a in add_file_idxs): - print( - "Pipeline order error: when using download-file with add-relationship, " - "add-relationship must come after add-file (so items are stored and have store+hash).\n" - "Example: download-file <...> | add-file -instance | add-relationship\n" - ) - return False - - return True - - @staticmethod - def _try_clear_pipeline_stop(ctx: Any) -> None: - try: - if hasattr(ctx, "clear_pipeline_stop"): - ctx.clear_pipeline_stop() - except Exception: - logger.exception("Failed to clear pipeline stop via ctx.clear_pipeline_stop") - - @staticmethod - def _maybe_seed_current_stage_table(ctx: Any) -> None: - try: - if hasattr(ctx, - "get_current_stage_table") and not ctx.get_current_stage_table(): - display_table = ( - ctx.get_display_table() if hasattr(ctx, - "get_display_table") else None - ) - if display_table: - ctx.set_current_stage_table(display_table) - else: - last_table = ( - ctx.get_last_result_table() - if hasattr(ctx, - "get_last_result_table") else None - ) - if last_table: - ctx.set_current_stage_table(last_table) - except Exception: - logger.exception("Failed to seed current_stage_table from display or last table") - - @staticmethod - def _maybe_apply_pending_pipeline_tail(ctx: Any, - stages: List[List[str]]) -> List[List[str]]: - try: - pending_tail = ( - ctx.get_pending_pipeline_tail() - if hasattr(ctx, - "get_pending_pipeline_tail") else [] - ) - pending_source = ( - ctx.get_pending_pipeline_source() - if hasattr(ctx, - "get_pending_pipeline_source") else None - ) - except Exception: - pending_tail = [] - pending_source = None - - try: - current_source = ( - ctx.get_current_stage_table_source_command() - if hasattr(ctx, - "get_current_stage_table_source_command") else None - ) - except Exception: - current_source = None - - try: - effective_source = current_source or ( - ctx.get_last_result_table_source_command() - if hasattr(ctx, - "get_last_result_table_source_command") else None - ) - except Exception: - effective_source = current_source - - selection_start = bool( - stages and stages[0] and stages[0][0].startswith("@") - ) - - def _tail_is_suffix(existing: List[List[str]], tail: List[List[str]]) -> bool: - if not tail or not existing: - return False - if len(tail) > len(existing): - return False - return existing[-len(tail):] == tail - - if pending_tail and selection_start: - if (pending_source is None) or (effective_source - and pending_source == effective_source): - # Only append the pending tail if the user hasn't already provided it. - if not _tail_is_suffix(stages, pending_tail): - stages = list(stages) + list(pending_tail) - try: - if hasattr(ctx, "clear_pending_pipeline_tail"): - ctx.clear_pending_pipeline_tail() - except Exception: - logger.exception("Failed to clear pending pipeline tail after appending pending tail") - else: - try: - if hasattr(ctx, "clear_pending_pipeline_tail"): - ctx.clear_pending_pipeline_tail() - except Exception: - logger.exception("Failed to clear pending pipeline tail (source mismatch branch)") - return stages - - def _apply_quiet_background_flag(self, config: Any) -> Any: - if isinstance(config, dict): - # This executor is used by both the REPL and the `pipeline` subcommand. - # Quiet/background mode is helpful for detached/background runners, but - # it suppresses interactive UX (like the pipeline Live progress UI). - try: - is_tty = bool(getattr(sys.stderr, "isatty", lambda: False)()) - except Exception: - is_tty = False - config["_quiet_background_output"] = not is_tty - return config - - @staticmethod - def _extract_first_stage_selection_tokens( - stages: List[List[str]], - ) -> tuple[List[List[str]], - List[int], - bool, - bool]: - first_stage_tokens = stages[0] if stages else [] - first_stage_selection_indices: List[int] = [] - first_stage_had_extra_args = False - first_stage_select_all = False - - if first_stage_tokens: - new_first_stage: List[str] = [] - for token in first_stage_tokens: - if token.startswith("@"): # selection - selection = _cli_parsing().SelectionSyntax.parse(token) - if selection is not None: - first_stage_selection_indices = [i - 1 for i in selection] - continue - if token == "@*": - first_stage_select_all = True - continue - new_first_stage.append(token) - - if new_first_stage: - stages = list(stages) - stages[0] = new_first_stage - if first_stage_selection_indices or first_stage_select_all: - first_stage_had_extra_args = True - elif first_stage_selection_indices or first_stage_select_all: - stages = list(stages) - stages.pop(0) - - return ( - stages, - first_stage_selection_indices, - first_stage_had_extra_args, - first_stage_select_all, - ) - - @staticmethod - def _apply_select_all_if_requested(ctx: Any, - indices: List[int], - select_all: bool) -> List[int]: - if not select_all: - return indices - try: - last_items = ctx.get_last_result_items() - except Exception: - last_items = None - if last_items: - return list(range(len(last_items))) - return indices - - @staticmethod - def _maybe_run_class_selector( - ctx: Any, - config: Any, - selected_items: list, - *, - stage_is_last: bool, - source_command: Any = None, - prefer_detail_fallback: bool = False, - ) -> bool: - if not stage_is_last: - return False - - candidates: list[str] = [] - seen: set[str] = set() - current_table = None - table_meta = None - table_type = "" - - def _add(value) -> None: - try: - text = str(value or "").strip().lower() - except Exception as exc: - logger.debug("Failed to normalize candidate value: %s", exc, exc_info=True) - return - if not text or text in seen: - return - seen.add(text) - candidates.append(text) - - try: - current_table = ctx.get_current_stage_table() or ctx.get_last_result_table() - _add( - current_table. - table if current_table and hasattr(current_table, - "table") else None - ) - if current_table and hasattr(current_table, "table"): - table_type = str(getattr(current_table, "table", "") or "").strip() - - # Prefer an explicit plugin hint from table metadata when available. - # This keeps @N selectors working even when row payloads don't carry a - # plugin key (or when they carry a table-type like tidal.album). - try: - meta = ( - current_table.get_table_metadata() - if current_table is not None and hasattr(current_table, "get_table_metadata") - else getattr(current_table, "table_metadata", None) - ) - except Exception: - meta = None - table_meta = meta if isinstance(meta, dict) else None - if isinstance(meta, dict): - _add(meta.get("plugin")) - except Exception: - logger.exception("Failed to inspect current_table/table metadata in _maybe_run_class_selector") - - for item in selected_items or []: - if isinstance(item, dict): - _add(item.get("plugin")) - _add(item.get("store")) - _add(item.get("table")) - else: - _add(getattr(item, "plugin", None)) - _add(getattr(item, "store", None)) - _add(getattr(item, "table", None)) - - try: - from PluginCore.registry import get_plugin, is_known_plugin_name - except Exception: - get_plugin = None # type: ignore - is_known_plugin_name = None # type: ignore - - # If we have a table-type like "tidal.album", also try its plugin prefix ("tidal") - # when that prefix is a registered plugin name. - if is_known_plugin_name is not None: - try: - for key in list(candidates): - if not isinstance(key, str): - continue - if "." not in key: - continue - if is_known_plugin_name(key): - continue - prefix = str(key).split(".", 1)[0].strip().lower() - if prefix and is_known_plugin_name(prefix): - _add(prefix) - except Exception: - logger.exception("Failed while computing plugin prefix heuristics in _maybe_run_class_selector") - - if get_plugin is not None: - for key in candidates: - try: - if is_known_plugin_name is not None and ( - not is_known_plugin_name(key)): - continue - except Exception: - logger.exception("is_known_plugin_name predicate failed for key %s; falling back", key) - try: - provider = get_plugin(key, config) - except Exception as exc: - logger.exception("Failed to load plugin '%s' during selector resolution: %s", key, exc) - continue - selector = getattr(provider, "selector", None) - if selector is None: - continue - try: - handled = bool( - selector(selected_items, - ctx=ctx, - stage_is_last=True) - ) - except Exception as exc: - logger.exception("%s selector failed during selection: %s", key, exc) - return True - if handled: - return True - - if prefer_detail_fallback: - detail_renderer = getattr(provider, "show_selection_details", None) - if callable(detail_renderer): - try: - detail_handled = bool( - detail_renderer( - selected_items, - ctx=ctx, - stage_is_last=True, - source_command=str(source_command or ""), - table_type=table_type, - table_metadata=table_meta, - ) - ) - except Exception as exc: - logger.exception("%s detail fallback failed during selection: %s", key, exc) - return True - if detail_handled: - return True - - @staticmethod - def _maybe_expand_plugin_selection( - selected_items: List[Any], - *, - ctx: Any, - config: Dict[str, Any], - stage_table: Any, - ) -> Optional[List[Any]]: - candidates: list[str] = [] - - def _add(value: Any) -> None: - text = str(value or "").strip().lower() - if text and text not in candidates: - candidates.append(text) - - table_type = None - try: - table_type = stage_table.table if stage_table is not None and hasattr(stage_table, "table") else None - except Exception: - table_type = None - _add(table_type) - - try: - meta = ( - stage_table.get_table_metadata() - if stage_table is not None and hasattr(stage_table, "get_table_metadata") - else getattr(stage_table, "table_metadata", None) - ) - except Exception: - meta = None - if isinstance(meta, dict): - _add(meta.get("plugin")) - - for item in selected_items or []: - if isinstance(item, dict): - _add(item.get("plugin")) - _add(item.get("table")) - _add(item.get("source")) - else: - _add(getattr(item, "plugin", None)) - _add(getattr(item, "table", None)) - _add(getattr(item, "source", None)) - - try: - from PluginCore.registry import get_plugin, is_known_plugin_name - except Exception: - return None - - for key in list(candidates): - if "." in key: - prefix = str(key).split(".", 1)[0].strip().lower() - if prefix and prefix not in candidates: - candidates.append(prefix) - - for key in candidates: - try: - if not is_known_plugin_name(key): - continue - except Exception: - continue - try: - plugin = get_plugin(key, config) - except Exception: - continue - if plugin is None: - continue - expand = getattr(plugin, "expand_selection", None) - if not callable(expand): - continue - try: - expanded = expand( - selected_items, - ctx=ctx, - stage_is_last=False, - table_type=str(table_type or ""), - ) - except Exception: - logger.exception("%s expand_selection failed", key) - return None - if expanded: - return list(expanded) - return None - - store_keys: list[str] = [] - for item in selected_items or []: - if isinstance(item, dict): - v = item.get("store") - else: - v = getattr(item, "store", None) - name = str(v or "").strip() - if name: - store_keys.append(name) - - if store_keys: - try: - from PluginCore.backend_registry import BackendRegistry - - backend_registry = BackendRegistry(config, suppress_debug=True) - _backend_names = list(backend_registry.list_backends() or []) - _backend_by_lower = { - str(n).lower(): str(n) - for n in _backend_names if str(n).strip() - } - for name in store_keys: - resolved_name = name - if not backend_registry.is_available(resolved_name): - resolved_name = _backend_by_lower.get(str(name).lower(), name) - if not backend_registry.is_available(resolved_name): - continue - backend = backend_registry[resolved_name] - selector = getattr(backend, "selector", None) - if selector is None: - continue - handled = bool( - selector(selected_items, - ctx=ctx, - stage_is_last=True) - ) - if handled: - return True - except Exception: - logger.exception("Failed while running store-based selector logic in _maybe_run_class_selector") - - return False - - @staticmethod - def _summarize_stage_text(stage_tokens: Sequence[str], limit: int = 140) -> str: - combined = " ".join(str(tok) for tok in stage_tokens if tok is not None).strip() - if not combined: - return "" - normalized = re.sub(r"\s+", " ", combined) - if len(normalized) <= limit: - return normalized - return normalized[:limit - 3].rstrip() + "..." - - @staticmethod - def _log_pipeline_event( - worker_manager: Any, - worker_id: Optional[str], - message: str, - ) -> None: - if not worker_manager or not worker_id or not message: - return - try: - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - except Exception: - timestamp = "" - if timestamp: - text = f"{timestamp} - PIPELINE - {message}" - else: - text = f"PIPELINE - {message}" - try: - worker_manager.append_stdout(worker_id, text + "\n", channel="log") - except Exception: - logger.exception("Failed to append pipeline event to worker stdout for %s", worker_id) - - @staticmethod - def _maybe_open_url_selection( - current_table: Any, - selected_items: list, - *, - stage_is_last: bool - ) -> bool: - if not stage_is_last: - return False - if not selected_items or len(selected_items) != 1: - return False - - table_type = "" - source_cmd = "" - try: - table_type = str(getattr(current_table, "table", "") or "").strip().lower() - except Exception: - table_type = "" - try: - source_cmd = ( - str(getattr(current_table, - "source_command", - "") or "").strip().replace("_", - "-").lower() - ) - except Exception: - source_cmd = "" - - if table_type != "url" and source_cmd != "get-url": - return False - - item = selected_items[0] - url = None - try: - from SYS.field_access import get_field - - url = get_field(item, "url") - except Exception: - try: - url = item.get("url") if isinstance(item, - dict - ) else getattr(item, - "url", - None) - except Exception: - url = None - - url_text = str(url or "").strip() - if not url_text: - return False - - try: - import webbrowser - - webbrowser.open(url_text, new=2) - return True - except Exception: - return False - - def _maybe_enable_background_notifier( - self, - worker_manager: Any, - config: Any, - pipeline_session: Any - ) -> None: - if not (pipeline_session and worker_manager and isinstance(config, dict)): - return - - session_worker_ids = config.get("_session_worker_ids") - if not session_worker_ids: - return - - try: - output_fn = self._toolbar_output - quiet_mode = bool(config.get("_quiet_background_output")) - terminal_only = quiet_mode and not output_fn - kwargs: Dict[str, - Any] = { - "session_worker_ids": session_worker_ids, - "only_terminal_updates": terminal_only, - "overlay_mode": bool(output_fn), - } - if output_fn: - kwargs["output"] = output_fn - from SYS.background_notifier import ensure_background_notifier - ensure_background_notifier(worker_manager, **kwargs) - except Exception: - logger.exception("Failed to enable background notifier for session_worker_ids=%r", session_worker_ids) - - @staticmethod - def _get_raw_stage_texts(ctx: Any) -> List[str]: - raw_stage_texts: List[str] = [] - try: - if hasattr(ctx, "get_current_command_stages"): - raw_stage_texts = ctx.get_current_command_stages() or [] - except Exception: - raw_stage_texts = [] - return raw_stage_texts - - def _maybe_apply_initial_selection( - self, - ctx: Any, - config: Any, - stages: List[List[str]], - *, - selection_indices: List[int], - first_stage_had_extra_args: bool, - worker_manager: Any, - pipeline_session: Any, - ) -> tuple[bool, - Any]: - if not selection_indices: - return True, None - - # ============================================================================ - # PHASE 1: Synchronize current stage table with display table - # ============================================================================ - # Selection should operate on the *currently displayed* selectable table. - # Some navigation flows (e.g. @.. back) can show a display table without - # updating current_stage_table. Provider selectors rely on current_stage_table - # to detect table type (e.g. tidal.album -> tracks), so sync it here. - display_table = None - try: - display_table = ( - ctx.get_display_table() if hasattr(ctx, "get_display_table") else None - ) - except Exception: - display_table = None - - current_stage_table = None - try: - current_stage_table = ( - ctx.get_current_stage_table() - if hasattr(ctx, "get_current_stage_table") else None - ) - except Exception: - current_stage_table = None - - try: - if display_table is not None and hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(display_table) - elif current_stage_table is None and hasattr(ctx, "set_current_stage_table"): - last_table = ( - ctx.get_last_result_table() - if hasattr(ctx, "get_last_result_table") else None - ) - if last_table is not None: - ctx.set_current_stage_table(last_table) - except Exception: - logger.exception("Failed to sync current_stage_table from display/last table in _maybe_apply_initial_selection") - - # ============================================================================ - # Helper functions for row action/args discovery (performance: inline caching) - # ============================================================================ - def _get_row_action(idx: int, items_cache: List[Any] | None = None) -> List[str] | None: - """Retrieve row selection_action from table or payload fallback.""" - try: - action = ctx.get_current_stage_table_row_selection_action(idx) - if action: - return [str(x) for x in action if x is not None] - except Exception: - pass - - # Fallback to serialized _selection_action in payload - if items_cache is None: - try: - items_cache = ctx.get_last_result_items() or [] - except Exception: - items_cache = [] - - if 0 <= idx < len(items_cache): - item = items_cache[idx] - if isinstance(item, dict): - candidate = item.get("_selection_action") - if isinstance(candidate, (list, tuple)): - return [str(x) for x in candidate if x is not None] - return None - - def _get_row_args(idx: int, items_cache: List[Any] | None = None) -> List[str] | None: - """Retrieve row selection_args from table or payload fallback.""" - try: - args = ctx.get_current_stage_table_row_selection_args(idx) - if args: - return [str(x) for x in args if x is not None] - except Exception: - pass - - # Fallback to serialized _selection_args in payload - if items_cache is None: - try: - items_cache = ctx.get_last_result_items() or [] - except Exception: - items_cache = [] - - if 0 <= idx < len(items_cache): - item = items_cache[idx] - if isinstance(item, dict): - candidate = item.get("_selection_args") - if isinstance(candidate, (list, tuple)): - return [str(x) for x in candidate if x is not None] - return None - - def _norm_cmd_name(value: Any) -> str: - return str(value or "").replace("_", "-").strip().lower() - - def _stage_file_action(stage_tokens: Sequence[Any]) -> str | None: - if not stage_tokens: - return None - head = _norm_cmd_name(stage_tokens[0]) - if head in {"add-file", "download-file", "delete-file"}: - return head - if head != "file": - return None - args = {_norm_cmd_name(t) for t in stage_tokens[1:]} - if "-add" in args or "--add" in args: - return "add-file" - if "-download" in args or "--download" in args or "-dl" in args or "--dl" in args: - return "download-file" - if "-delete" in args or "--delete" in args or "-del" in args or "--del" in args: - return "delete-file" - return None - - # ============================================================================ - # PHASE 2: Parse source command and table metadata - # ============================================================================ - source_cmd = None - source_args_raw = None - try: - source_cmd = ctx.get_current_stage_table_source_command() - source_args_raw = ctx.get_current_stage_table_source_args() - except Exception: - source_cmd = None - source_args_raw = None - - if isinstance(source_args_raw, str): - source_args: List[str] = [source_args_raw] - elif isinstance(source_args_raw, list): - source_args = [str(x) for x in source_args_raw if x is not None] - else: - source_args = [] - - current_table = None - try: - current_table = ctx.get_current_stage_table() - except Exception: - current_table = None - table_type = ( - current_table.table if current_table and hasattr(current_table, - "table") else None - ) - - # ============================================================================ - # PHASE 3: Handle command expansion for @N syntax - # ============================================================================ - command_expanded = False - example_selector_triggered = False - normalized_source_cmd = str(source_cmd or "").replace("_", "-").strip().lower() - prefer_row_action = False - preferred_row_action = None - - if normalized_source_cmd in HELP_EXAMPLE_SOURCE_COMMANDS and selection_indices: - try: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - except Exception: - row_args = None - tokens: List[str] = [] - if isinstance(row_args, list) and row_args: - tokens = [str(x) for x in row_args if x is not None] - if tokens: - stage_groups = _split_pipeline_tokens(tokens) - if stage_groups: - for stage in reversed(stage_groups): - stages.insert(0, stage) - selection_indices = [] - command_expanded = True - example_selector_triggered = True - - if not example_selector_triggered: - if table_type in {"youtube", - "soulseek"}: - command_expanded = False - elif source_cmd == "search-file" and source_args and "youtube" in source_args: - command_expanded = False - else: - selected_row_args: List[str] = [] - skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0 - if len(selection_indices) == 1 and not stages: - try: - row_action = _get_row_action(selection_indices[0]) - except Exception: - row_action = None - if row_action: - stages.insert(0, list(row_action)) - return True, None - # Command expansion via @N: - # - Default behavior: expand ONLY for single-row selections. - # - Special case: allow multi-row expansion for add-file directory tables by - # combining selected rows into one comma-separated positional source token. - if source_cmd and not skip_pipe_expansion and not prefer_row_action: - src = str(source_cmd).replace("_", "-").strip().lower() - - if src == "add-file" and selection_indices: - row_args_list: List[List[str]] = [] - for idx in selection_indices: - try: - row_args = ctx.get_current_stage_table_row_selection_args( - idx - ) - except Exception: - row_args = None - if isinstance(row_args, list) and row_args: - row_args_list.append( - [str(x) for x in row_args if x is not None] - ) - - # Combine `[]` from each row into one positional source token. - paths: List[str] = [] - can_merge = bool(row_args_list) and ( - len(row_args_list) == len(selection_indices) - ) - if can_merge: - for ra in row_args_list: - if len(ra) == 1: - p = str(ra[0]).strip() - if p: - paths.append(p) - else: - can_merge = False - break - - if can_merge and paths: - selected_row_args.append(",".join(paths)) - elif len(selection_indices) == 1 and row_args_list: - selected_row_args.extend(row_args_list[0]) - else: - # Only perform @N command expansion for *single-item* selections. - # For multi-item selections (e.g. @*, @1-5), expanding to one row - # would silently drop items. In those cases we pipe items downstream. - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - if row_args: - selected_row_args.extend(row_args) - - if selected_row_args and not stages: - if isinstance(source_cmd, list): - cmd_list: List[str] = [str(x) for x in source_cmd if x is not None] - elif isinstance(source_cmd, str): - cmd_list = [source_cmd] - else: - cmd_list = [] - - # IMPORTANT: Put selected row args *before* source_args. - # Rationale: The cmdlet argument parser treats the *first* unknown - # token as a positional value (e.g., URL). If `source_args` - # contain unknown flags (like a removed legacy flag that download-file does - # not declare), they could be misinterpreted as the positional - # URL argument and cause attempts to download strings like - # not accept). By placing selection args - # first we ensure the intended URL/selection token is parsed - # as the positional URL and avoid this class of parsing errors. - expanded_stage: List[str] = cmd_list + selected_row_args + source_args - - stages.insert(0, expanded_stage) - - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", - ) - except Exception: - logger.exception("Failed to record pipeline log step for @N expansion (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - elif selected_row_args and stages: - pass - - stage_table = None - try: - stage_table = ctx.get_current_stage_table() - except Exception: - stage_table = None - - display_table = None - try: - display_table = ( - ctx.get_display_table() if hasattr(ctx, - "get_display_table") else None - ) - except Exception: - display_table = None - - if not stage_table and display_table is not None: - stage_table = display_table - if not stage_table: - try: - stage_table = ctx.get_last_result_table() - except Exception: - stage_table = None - - # ==================================================================== - # PHASE 4: Retrieve and filter items from current result set - # ==================================================================== - # Cache items_list to avoid redundant lookups in helper functions below. - # Priority: display items (from overlays like get-metadata) > last result items - try: - items_list = ctx.get_last_result_items() or [] - except Exception as exc: - debug(f"@N: Exception getting items_list: {exc}") - items_list = [] - resolved_items = items_list if items_list else [] - if items_list: - filtered = [ - resolved_items[i] for i in selection_indices - if 0 <= i < len(resolved_items) - ] - if selection_indices: - if len(selection_indices) == 1: - selection_label = f"@{selection_indices[0] + 1}" - else: - selection_label = "@{" + ",".join(str(idx + 1) for idx in selection_indices) + "}" - else: - selection_label = "@selection" - _emit_selection_debug_panel( - selection_token=selection_label, - selection_indices=selection_indices, - item_count=len(items_list), - filtered_count=len(filtered), - stage_table_present=(stage_table is not None), - display_table_present=(display_table is not None), - stage_is_last=(not stages), - row_action=preferred_row_action, - downstream_stages=stages, - mode=("row_action" if preferred_row_action else "selection"), - ) - if not filtered: - print("No items matched selection in pipeline\n") - return False, None - - if stages: - expanded = PipelineExecutor._maybe_expand_plugin_selection( - filtered, - ctx=ctx, - config=config, - stage_table=stage_table, - ) - if expanded: - filtered = expanded - - if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(not stages), - source_command=source_cmd, - prefer_detail_fallback=bool(prefer_row_action and not stages and len(selection_indices) == 1)): - return False, None - - from SYS.pipe_object import coerce_to_pipe_object - - filtered_pipe_objs = [coerce_to_pipe_object(item) for item in filtered] - piped_result = ( - filtered_pipe_objs - if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] - ) - - if pipeline_session and worker_manager: - try: - selection_parts = [f"@{i+1}" for i in selection_indices] - worker_manager.log_step( - pipeline_session.worker_id, - f"Applied @N selection {' | '.join(selection_parts)}", - ) - except Exception: - logger.exception("Failed to record Applied @N selection log step (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - - # Auto-insert downloader stages for plugin tables. - try: - current_table = ctx.get_current_stage_table() - if current_table is None and hasattr(ctx, "get_display_table"): - current_table = ctx.get_display_table() - if current_table is None: - current_table = ctx.get_last_result_table() - except Exception: - logger.exception("Failed to determine current_table for selection auto-insert; defaulting to None") - current_table = None - table_type_hint = None - try: - raw_table_type = ( - stage_table.table - if stage_table is not None and hasattr(stage_table, "table") else None - ) - if isinstance(raw_table_type, str) and raw_table_type.strip(): - table_type_hint = raw_table_type - except Exception: - table_type_hint = None - table_type = None - try: - if isinstance(table_type_hint, str) and table_type_hint.strip(): - table_type = table_type_hint - else: - table_type = ( - current_table.table - if current_table and hasattr(current_table, "table") else None - ) - except Exception: - logger.exception("Failed to compute table_type from current_table; using fallback attribute access") - table_type = ( - current_table.table - if current_table and hasattr(current_table, "table") else None - ) - - def _norm_cmd(name: Any) -> str: - return str(name or "").replace("_", "-").strip().lower() - - auto_stage = None - if isinstance(table_type, str) and table_type: - try: - from PluginCore.registry import selection_auto_stage_for_table - - auto_stage = selection_auto_stage_for_table(table_type) - except Exception: - auto_stage = None - - source_cmd_for_selection = None - source_args_for_selection: List[str] = [] - try: - source_cmd_for_selection = ( - ctx.get_current_stage_table_source_command() - or ctx.get_last_result_table_source_command() - ) - source_args_for_selection = ( - ctx.get_current_stage_table_source_args() - or ctx.get_last_result_table_source_args() - or [] - ) - except Exception: - source_cmd_for_selection = None - source_args_for_selection = [] - - if not stages and selection_indices and source_cmd_for_selection: - src_norm = _norm_cmd_name(source_cmd_for_selection) - if src_norm in {".worker", "worker", "workers"}: - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = _get_row_args(idx, items_list) - if row_args: - stages.append( - [str(source_cmd_for_selection)] - + row_args - + [str(x) for x in source_args_for_selection if x is not None] - ) - - def _apply_row_action_to_stage(stage_idx: int) -> bool: - """Apply row selection_action to a specific stage, replacing it.""" - if not selection_indices or len(selection_indices) != 1: - return False - row_action = _get_row_action(selection_indices[0], items_list) - if not row_action: - return False - if 0 <= stage_idx < len(stages): - stages[stage_idx] = row_action - return True - return False - - # ==================================================================== - # PHASE 5: Auto-insert stages based on table type and user selection - # ==================================================================== - if not stages: - if isinstance(table_type, str) and table_type.startswith("metadata."): - print("Auto-applying metadata selection via metadata -get") - stages.append(["metadata", "-get"]) - elif auto_stage: - try: - print(f"Auto-running selection via {auto_stage[0]}") - except Exception: - logger.exception("Failed to print auto-run selection message for %s", auto_stage[0]) - # Append the auto stage now. If the user also provided a selection - # (e.g., @1 | add-file ...), we want to attach the row selection - # args *to the auto-inserted stage* so the download command receives - # the selected row information immediately. - stages.append(list(auto_stage)) - debug(f"Inserted auto stage before row action: {stages[-1]}") - - # Attach selection args to auto stage - if selection_indices: - try: - if not _apply_row_action_to_stage(len(stages) - 1): - # Only support single-row selection for auto-attach here - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = _get_row_args(idx, items_list) - if row_args: - # Place selection args before any existing source args - inserted = stages[-1] - if inserted: - cmd = inserted[0] - tail = [str(x) for x in inserted[1:]] - stages[-1] = [cmd] + row_args + tail - except Exception: - logger.exception("Failed to attach selection args to auto-inserted stage") - - # Look for row_action in payload if still no stages - if not stages and selection_indices and len(selection_indices) == 1: - row_action = _get_row_action(selection_indices[0], items_list) - if row_action: - stages.append(row_action) - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - f"@N applied row action -> {' '.join(row_action)}", - ) - except Exception: - logger.exception("Failed to record pipeline log step for applied row action (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - else: - first_cmd = stages[0][0] if stages and stages[0] else None - first_cmd_norm = _norm_cmd_name(first_cmd) - - inserted_provider_download = False - if _stage_file_action(stages[0]) == "add-file": - # If selected rows advertise an explicit download-file action, - # run download before add-file so add-file receives local files. - if len(selection_indices) == 1: - row_action = _get_row_action(selection_indices[0], items_list) - if row_action and _stage_file_action(row_action) == "download-file": - stages.insert(0, [str(x) for x in row_action if x is not None]) - inserted_provider_download = True - debug("Auto-inserting row download-file action before add-file") - - # Multi-selection fallback: if any selected row declares a - # download-file action, insert a generic download-file stage. - # This keeps plugin-specific behavior in plugin metadata. - if (not inserted_provider_download) and len(selection_indices) > 1: - try: - has_download_row_action = False - for idx in selection_indices: - row_action = _get_row_action(idx, items_list) - if row_action and _stage_file_action(row_action) == "download-file": - has_download_row_action = True - break - if has_download_row_action: - stages.insert(0, ["file", "-download"]) - inserted_provider_download = True - debug("Auto-inserting download-file before add-file for provider selection") - except Exception: - pass - - if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in ( - "metadata", - "tag", - ".pipe", - ".mpv", - ): - print("Auto-inserting metadata -get after metadata selection") - stages.insert(0, ["metadata", "-get"]) - elif auto_stage: - first_cmd_norm = _norm_cmd_name(stages[0][0] if stages and stages[0] else None) - auto_cmd_norm = _norm_cmd_name(auto_stage[0]) - if first_cmd_norm not in (auto_cmd_norm, ".pipe", ".mpv"): - debug(f"Auto-inserting {auto_cmd_norm} after selection") - # Insert the auto stage before the user-specified stage - # Note: Do NOT append source_args here - they are search tokens from - # the previous stage and should not be passed to the downloader. - stages.insert(0, list(auto_stage)) - debug(f"Inserted auto stage before existing pipeline: {stages[0]}") - - # If a selection is present, attach the row selection args to the - # newly-inserted stage so the download stage runs with the - # selected row information. - if selection_indices: - try: - if not _apply_row_action_to_stage(0): - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = _get_row_args(idx, items_list) - if row_args: - inserted = stages[0] - if inserted: - cmd = inserted[0] - tail = [str(x) for x in inserted[1:]] - stages[0] = [cmd] + row_args + tail - except Exception: - logger.exception("Failed to attach selection args to inserted auto stage (alternate branch)") - - # After inserting/appending an auto-stage, continue processing so later - # selection-expansion logic can still run (e.g., for example selectors). - if (not stages) and selection_indices and len(selection_indices) == 1: - # Selection-only invocation (e.g. user types @1 with no pipe). - # Show the item details panel so selection feels actionable. - try: - selected_item = filtered[0] if filtered else None - if selected_item is not None and not isinstance(selected_item, dict): - to_dict = getattr(selected_item, "to_dict", None) - if callable(to_dict): - selected_item = to_dict() - if isinstance(selected_item, dict): - from SYS.rich_display import render_item_details_panel - - render_item_details_panel(selected_item) - try: - ctx.set_last_result_items_only([selected_item]) - except Exception: - pass - except Exception: - logger.exception("Failed to render selection-only item details") - return True, piped_result - else: - debug(f"@N: No items to select from (items_list empty)") - print("No previous results to select from\n") - return False, None - - return True, None - - @staticmethod - def _maybe_start_live_progress(config: Any, - stages: List[List[str]]) -> tuple[Any, - Dict[int, - int]]: - progress_ui = None - pipe_index_by_stage: Dict[int, - int] = {} - - try: - quiet_mode = ( - bool(config.get("_quiet_background_output")) - if isinstance(config, - dict) else False - ) - except Exception: - quiet_mode = False - - try: - import sys as _sys - - if (not quiet_mode) and bool(getattr(_sys.stderr, - "isatty", lambda: False)()): - from SYS.models import PipelineLiveProgress - - pipe_stage_indices: List[int] = [] - pipe_labels: List[str] = [] - for idx, stage_tokens in enumerate(stages): - if not stage_tokens: - continue - name = str(stage_tokens[0]).replace("_", "-").lower() - if name == "@" or name.startswith("@"): - continue - - # add-file directory selector stage: avoid Live progress so the - # selection table renders cleanly. - if _stage_file_action(stage_tokens) == "add-file" or name in {"add_file"}: - try: - from pathlib import Path as _Path - - toks = list(stage_tokens[1:]) - i = 0 - while i < len(toks): - t = str(toks[i]) - low = t.lower().strip() - if low in {"-path", - "--path", - "-p"} and i + 1 < len(toks): - nxt = str(toks[i + 1]) - if nxt and ("," not in nxt): - p = _Path(nxt) - if p.exists() and p.is_dir(): - name = "" # mark as skipped - break - i += 2 - continue - i += 1 - except Exception: - logger.exception("Failed to inspect add-file stage tokens for potential directory; skipping Live progress") - if not name: - continue - # Display-only: avoid Live progress for relationship viewing. - # This keeps `@1 | get-relationship` clean and prevents progress UI - # from interfering with Rich tables/panels. - if name in {"get-relationship", - "get-rel"}: - continue - if name in {"get-metadata", - "meta"}: - continue - # `.pipe` (MPV) is an interactive launcher; disable pipeline Live progress - # for it because it doesn't meaningfully "complete" (mpv may keep running) - # and Live output interferes with MPV playlist UI. - if name in {".pipe", ".mpv"}: - continue - # `.matrix` uses a two-phase picker (@N then .matrix -send). Pipeline Live - # progress can linger across those phases and interfere with interactive output. - if name == ".matrix": - continue - # `delete-file` prints a Rich table directly; Live progress interferes and - # can truncate/overwrite the output. - if _stage_file_action(stage_tokens) == "delete-file" or name in {"del-file"}: - continue - pipe_stage_indices.append(idx) - pipe_labels.append(name) - - if pipe_labels: - progress_ui = PipelineLiveProgress(pipe_labels, enabled=True) - progress_ui.start() - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(progress_ui) - if hasattr(_pipeline_ctx, "get_progress_event_callback"): - progress_cb = _pipeline_ctx.get_progress_event_callback() - if callable(progress_cb) and hasattr(progress_ui, "set_event_callback"): - progress_ui.set_event_callback(progress_cb) - except Exception: - logger.exception("Failed to register PipelineLiveProgress with pipeline context") - pipe_index_by_stage = { - stage_idx: pipe_idx - for pipe_idx, stage_idx in enumerate(pipe_stage_indices) - } - except Exception: - progress_ui = None - pipe_index_by_stage = {} - - return progress_ui, pipe_index_by_stage - - def execute_tokens(self, tokens: List[str]) -> None: - from cmdlet import REGISTRY - ctx = sys.modules[__name__] - - try: - try: - from SYS.logger import debug_panel - - debug_panel( - "Pipeline execution", - [ - ("command", " ".join(str(tok) for tok in tokens)), - ("token_count", len(tokens)), - ], - ) - except Exception: - debug(f"execute_tokens: tokens={tokens}") - self._try_clear_pipeline_stop(ctx) - - # REPL guard: stage-local tables should not persist across independent - # commands. Selection stages can always seed from last/display tables. - try: - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(None) - except Exception: - logger.exception("Failed to clear current_stage_table in execute_tokens") - - # Preflight (URL-duplicate prompts, etc.) should be cached within a single - # pipeline run, not across independent pipelines. - try: - ctx.store_value("preflight", - {}) - except Exception: - logger.exception("Failed to set preflight cache in execute_tokens") - - stages = self._split_stages(tokens) - if not stages: - print("Invalid pipeline syntax\n") - return - self._maybe_seed_current_stage_table(ctx) - stages = self._maybe_apply_pending_pipeline_tail(ctx, stages) - config = self._load_config() - config = self._apply_quiet_background_flag(config) - - ( - stages, - first_stage_selection_indices, - first_stage_had_extra_args, - first_stage_select_all, - ) = self._extract_first_stage_selection_tokens(stages) - first_stage_selection_indices = self._apply_select_all_if_requested( - ctx, - first_stage_selection_indices, - first_stage_select_all - ) - - piped_result: Any = None - worker_manager = _worker().WorkerManagerRegistry.ensure(config) - pipeline_text = " | ".join(" ".join(stage) for stage in stages) - pipeline_session = _worker().WorkerStages.begin_pipeline( - worker_manager, - pipeline_text=pipeline_text, - config=config - ) - if pipeline_session and worker_manager: - self._log_pipeline_event( - worker_manager, - pipeline_session.worker_id, - f"Pipeline start: {pipeline_text or '(empty pipeline)'}", - ) - raw_stage_texts = self._get_raw_stage_texts(ctx) - self._maybe_enable_background_notifier( - worker_manager, - config, - pipeline_session - ) - - pipeline_status = "completed" - pipeline_error = "" - - progress_ui = None - pipe_index_by_stage: Dict[int, - int] = {} - - ok, initial_piped = self._maybe_apply_initial_selection( - ctx, - config, - stages, - selection_indices=first_stage_selection_indices, - first_stage_had_extra_args=first_stage_had_extra_args, - worker_manager=worker_manager, - pipeline_session=pipeline_session, - ) - if not ok: - return - if initial_piped is not None: - piped_result = initial_piped - - # REPL guard: prevent add-relationship before add-file for download-file pipelines. - if not self._validate_download_file_relationship_order(stages): - pipeline_status = "failed" - pipeline_error = "Invalid pipeline order" - return - - # ------------------------------------------------------------------ - # Multi-level pipeline progress (pipes = stages, tasks = items) - # ------------------------------------------------------------------ - progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages) - - for stage_index, stage_tokens in enumerate(stages): - if not stage_tokens: - continue - - raw_stage_name = str(stage_tokens[0]) - cmd_name = raw_stage_name.replace("_", "-").lower() - stage_args = stage_tokens[1:] - - if cmd_name == "@": - # Special-case metadata -get tables: `@ | metadata -add ...` should target the - # underlying file subject once, not each emitted TagItem row. - try: - next_cmd = None - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = str(stages[stage_index + 1][0]).replace("_", "-").strip().lower() - - current_table = None - try: - current_table = ctx.get_current_stage_table() or ctx.get_last_result_table() - except Exception: - current_table = None - - source_cmd = str(getattr(current_table, "source_command", "") or "").replace("_", "-").strip().lower() - is_get_tag_table = source_cmd == "metadata" - - if is_get_tag_table and next_cmd == "metadata": - subject = ctx.get_last_result_subject() - if subject is not None: - next_args = [ - str(x).replace("_", "-").strip().lower() - for x in (stages[stage_index + 1][1:] if stage_index + 1 < len(stages) else []) - ] - if "-add" not in next_args and "--add" not in next_args: - # Only apply this flattening for explicit tag-add pipelines. - pass - else: - piped_result = subject - try: - subject_items = subject if isinstance(subject, list) else [subject] - ctx.set_last_items(subject_items) - except Exception: - logger.exception("Failed to set last_items from tag subject during @ handling") - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used metadata table subject for metadata -add" - ) - except Exception: - logger.exception("Failed to record pipeline log step for '@ used metadata table subject for metadata -add' (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - continue - except Exception: - logger.exception("Failed to evaluate tag @ subject special-case") - - # Prefer piping the last emitted/visible items (e.g. add-file results) - # over the result-table subject. The subject can refer to older context - # (e.g. a playlist row) and may not contain store+hash. - last_items = None - try: - last_items = ctx.get_last_result_items() - except Exception: - last_items = None - - if last_items: - from SYS.pipe_object import coerce_to_pipe_object - - try: - pipe_items = [ - coerce_to_pipe_object(x) for x in list(last_items) - ] - except Exception: - pipe_items = list(last_items) - piped_result = pipe_items if len(pipe_items - ) > 1 else pipe_items[0] - try: - ctx.set_last_items(pipe_items) - except Exception: - logger.exception("Failed to set last items after @ selection") - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used last result items" - ) - except Exception: - logger.exception("Failed to record pipeline log step for '@ used last result items' (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - continue - - subject = ctx.get_last_result_subject() - if subject is None: - print("No current result context available for '@'\n") - pipeline_status = "failed" - pipeline_error = "No result items/subject for @" - return - piped_result = subject - try: - subject_items = subject if isinstance(subject, - list) else [subject] - ctx.set_last_items(subject_items) - except Exception: - logger.exception("Failed to set last_items from subject during @ handling") - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used current table subject" - ) - except Exception: - logger.exception("Failed to record pipeline log step for '@ used current table subject' (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - continue - - if cmd_name.startswith("@"): # selection stage - selection_token = raw_stage_name - selection = _cli_parsing().SelectionSyntax.parse(selection_token) - filter_spec = _cli_parsing().SelectionFilterSyntax.parse(selection_token) - is_select_all = selection_token.strip() == "@*" - if selection is None and filter_spec is None and not is_select_all: - print(f"Invalid selection: {selection_token}\n") - pipeline_status = "failed" - pipeline_error = f"Invalid selection {selection_token}" - return - - selected_indices = [] - # Prefer selecting from the last selectable *table* (search/playlist) - # rather than from display-only emitted items, unless we're explicitly - # selecting from an overlay table. - display_table = None - try: - display_table = ( - ctx.get_display_table() - if hasattr(ctx, - "get_display_table") else None - ) - except Exception: - display_table = None - - stage_table = ctx.get_current_stage_table() - # Selection should operate on the table the user sees. - # If a display overlay table exists, force it as the current-stage table - # so provider selectors (e.g. tidal.album -> tracks) behave consistently. - try: - if display_table is not None and hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(display_table) - stage_table = display_table - except Exception: - logger.exception("Failed to set current_stage_table from display table during selection processing") - - if not stage_table and display_table is not None: - stage_table = display_table - if not stage_table: - stage_table = ctx.get_last_result_table() - - try: - if hasattr(ctx, "debug_table_state"): - ctx.debug_table_state(f"selection {selection_token}") - except Exception: - logger.exception("Failed to debug_table_state during selection %s", selection_token) - - if display_table is not None and stage_table is display_table: - items_list = ctx.get_last_result_items() or [] - else: - if hasattr(ctx, "get_last_selectable_result_items"): - items_list = ctx.get_last_selectable_result_items( - ) or [] - else: - items_list = ctx.get_last_result_items() or [] - - if is_select_all: - selected_indices = list(range(len(items_list))) - elif filter_spec is not None: - selected_indices = [ - i for i, item in enumerate(items_list) - if _cli_parsing().SelectionFilterSyntax.matches(item, filter_spec) - ] - else: - selected_indices = [i - 1 for i in selection] # type: ignore[arg-type] - - resolved_items = items_list if items_list else [] - filtered = [ - resolved_items[i] for i in selected_indices - if 0 <= i < len(resolved_items) - ] - # Debug: show selection resolution and sample payload info - try: - debug(f"Selection {selection_token} -> resolved_indices={selected_indices} filtered_count={len(filtered)}") - if filtered: - sample = filtered[0] - if isinstance(sample, dict): - debug(f"Selection sample: hash={sample.get('hash')} store={sample.get('store')} _selection_args={sample.get('_selection_args')} _selection_action={sample.get('_selection_action')}") - else: - try: - debug(f"Selection sample object: provider={getattr(sample, 'provider', None)} store={getattr(sample, 'store', None)}") - except Exception: - logger.exception("Failed to debug selection sample object") - except Exception: - logger.exception("Failed to produce selection debug sample for token %s", selection_token) - - if not filtered: - print("No items matched selection\n") - pipeline_status = "failed" - pipeline_error = "Empty selection" - return - - # Filter UX: if the stage token is a filter and it's terminal, - # render a filtered table overlay rather than selecting/auto-downloading. - stage_is_last = (stage_index + 1 >= len(stages)) - if filter_spec is not None and stage_is_last: - try: - base_table = stage_table - if base_table is None: - base_table = ctx.get_last_result_table() - - if base_table is not None and hasattr(base_table, "copy_with_title"): - new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") - else: - new_table = _result_table()(getattr(base_table, "title", "") if base_table is not None else "Results") - - try: - if base_table is not None and getattr(base_table, "table", None): - new_table.set_table(str(getattr(base_table, "table"))) - except Exception: - logger.exception("Failed to set table on new_table for filter overlay") - - try: - # Attach a one-line header so users see the active filter. - safe = str(selection_token)[1:].strip() - new_table.set_header_line(f'filter: "{safe}"') - except Exception: - logger.exception("Failed to set header line for filter overlay for token %s", selection_token) - - for item in filtered: - new_table.add_result(item) - - try: - ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject()) - except Exception: - logger.exception("Failed to set last_result_table_overlay for filter selection") - - try: - from SYS.rich_display import stdout_console - stdout_console().print() - stdout_console().print(new_table) - except Exception: - logger.exception("Failed to render filter overlay to stdout_console") - except Exception: - logger.exception("Failed while rendering filter overlay for selection %s", selection_token) - continue - - # UX: selecting a single URL row from get-url tables should open it. - # Only do this when the selection stage is terminal to avoid surprising - # side-effects in pipelines like `@1 | download-file`. - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - if (not is_select_all) and (len(filtered) == 1): - try: - PipelineExecutor._maybe_open_url_selection( - current_table, - filtered, - stage_is_last=(stage_index + 1 >= len(stages)), - ) - except Exception: - logger.exception("Failed to open URL selection for table %s", getattr(current_table, 'table', None)) - - if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(stage_index + 1 >= len(stages))): - return - - # Special case: selecting multiple metadata tag rows and piping into metadata -delete - # should batch into a single operation (one backend call). - next_cmd = None - next_args: List[str] = [] - try: - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = str(stages[stage_index + 1][0] - ).replace("_", - "-").lower() - next_args = [ - str(x).replace("_", "-").strip().lower() - for x in stages[stage_index + 1][1:] - ] - except Exception: - logger.exception("Failed to determine next_cmd during selection expansion for stage_index %s", stage_index) - next_cmd = None - next_args = [] - - def _is_tag_row(obj: Any) -> bool: - try: - if (hasattr(obj, - "__class__") - and obj.__class__.__name__ == "TagItem" - and hasattr(obj, - "tag_name")): - return True - except Exception: - logger.exception("Failed to inspect TagItem object while checking _is_tag_row") - try: - if isinstance(obj, dict) and obj.get("tag_name"): - return True - except Exception: - logger.exception("Failed to inspect dict tag_name while checking _is_tag_row") - return False - - if (next_cmd == "tag" and ("-delete" in next_args or "--delete" in next_args) and len(filtered) > 1 - and all(_is_tag_row(x) for x in filtered)): - from SYS.field_access import get_field - - tags: List[str] = [] - first_hash = None - first_store = None - first_path = None - for item in filtered: - tag_name = get_field(item, "tag_name") - if tag_name: - tags.append(str(tag_name)) - if first_hash is None: - first_hash = get_field(item, "hash") - if first_store is None: - first_store = get_field(item, "store") - if first_path is None: - first_path = get_field(item, - "path") or get_field( - item, - "target" - ) - - if tags: - grouped = { - "table": "tag.selection", - "media_kind": "tag", - "hash": first_hash, - "store": first_store, - "path": first_path, - "tag": tags, - } - piped_result = grouped - continue - - from SYS.pipe_object import coerce_to_pipe_object - - filtered_pipe_objs = [ - coerce_to_pipe_object(item) for item in filtered - ] - piped_result = ( - filtered_pipe_objs - if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] - ) - - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - table_type = ( - current_table.table - if current_table and hasattr(current_table, - "table") else None - ) - - def _norm_stage_cmd(name: Any) -> str: - return str(name or "").replace("_", "-").strip().lower() - - next_cmd = None - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = _norm_stage_cmd(stages[stage_index + 1][0]) - - auto_stage = None - if isinstance(table_type, str) and table_type: - try: - from PluginCore.registry import selection_auto_stage_for_table - - # Preserve historical behavior: only forward selection-stage args - # to the auto stage when we are appending a new last stage. - at_end = bool(stage_index + 1 >= len(stages)) - auto_stage = selection_auto_stage_for_table( - table_type, - stage_args if at_end else None, - ) - except Exception: - auto_stage = None - - # Auto-insert downloader stages for provider tables. - # IMPORTANT: do not auto-download for filter selections; they may match many rows. - if filter_spec is None: - if stage_index + 1 >= len(stages): - if auto_stage: - try: - print(f"Auto-running selection via {auto_stage[0]}") - except Exception: - logger.exception("Failed to print auto-run selection message for %s", auto_stage[0]) - stages.append(list(auto_stage)) - else: - if auto_stage: - auto_cmd = _norm_stage_cmd(auto_stage[0]) - if next_cmd not in (auto_cmd, ".pipe", ".mpv"): - debug(f"Auto-inserting {auto_cmd} after selection") - stages.insert(stage_index + 1, list(auto_stage)) - continue - - cmd_fn = REGISTRY.get(cmd_name) - try: - mod = import_cmd_module(cmd_name, reload_loaded=True) - data = getattr(mod, "CMDLET", None) if mod else None - if data and hasattr(data, "exec") and callable(getattr(data, "exec")): - from SYS.cmdlet_spec import collect_registered_cmdlet_names - - run_fn = getattr(data, "exec") - for registered_name in collect_registered_cmdlet_names(data, fallback_name=cmd_name): - REGISTRY[registered_name] = run_fn - cmd_fn = run_fn - except Exception: - pass - - if not cmd_fn: - try: - mod = import_cmd_module(cmd_name) - data = getattr(mod, "CMDLET", None) if mod else None - if data and hasattr(data, "exec") and callable(getattr(data, "exec")): - run_fn = getattr(data, "exec") - REGISTRY[cmd_name] = run_fn - cmd_fn = run_fn - except Exception: - cmd_fn = None - - if not cmd_fn: - print(f"Unknown command: {cmd_name}\n") - pipeline_status = "failed" - pipeline_error = f"Unknown command: {cmd_name}" - return - - try: - from SYS.models import PipelineStageContext - - pipe_idx = pipe_index_by_stage.get(stage_index) - - output_table: Any | None = None - pre_stage_table: Any | None = None - pre_last_result_table: Any | None = None - session = _worker().WorkerStages.begin_stage( - worker_manager, - cmd_name=cmd_name, - stage_tokens=stage_tokens, - config=config, - command_text=pipeline_text if pipeline_text else " ".join(stage_tokens), - ) - try: - stage_ctx = PipelineStageContext( - stage_index=stage_index, - total_stages=len(stages), - pipe_index=pipe_idx, - worker_id=session.worker_id if session else None, - on_emit=(lambda x: progress_ui.on_emit(pipe_idx, x)) - if progress_ui is not None and pipe_idx is not None else None, - ) - - # Set context for the current run - ctx.set_stage_context(stage_ctx) - ctx.set_current_cmdlet_name(cmd_name) - ctx.set_current_stage_text(" ".join(stage_tokens)) - ctx.clear_emits() - - if progress_ui is not None and pipe_idx is not None: - # Start the pipe task in the UI. For most cmdlets we assume 1 item - # initially; cmdlets that process multiple items (like search) - # should call begin_pipe themselves with the actual count. - progress_ui.begin_pipe(pipe_idx, total_items=1) - - if stage_index + 1 >= len(stages): - try: - pre_stage_table = ( - ctx.get_current_stage_table() - if hasattr(ctx, "get_current_stage_table") else None - ) - except Exception: - pre_stage_table = None - try: - pre_last_result_table = ( - ctx.get_last_result_table() - if hasattr(ctx, "get_last_result_table") else None - ) - except Exception: - pre_last_result_table = None - - # RUN THE CMDLET - ret_code = cmd_fn(piped_result, stage_args, config) - if ret_code is not None: - try: - normalized_ret = int(ret_code) - except Exception: - normalized_ret = 0 - if normalized_ret != 0: - pipeline_status = "failed" - pipeline_error = f"Stage '{cmd_name}' failed with exit code {normalized_ret}" - return - - # Terminal pipeline stages need to render overlay tables and also - # newly produced standard result tables from row actions like - # `.config -browse ...`, because there is no outer CLI render pass. - output_table = None - if stage_index + 1 >= len(stages): - try: - output_table = ( - ctx.get_display_table() - if hasattr(ctx, "get_display_table") else None - ) - except Exception: - output_table = None - - if output_table is None: - current_stage_table = None - last_result_table = None - try: - current_stage_table = ( - ctx.get_current_stage_table() - if hasattr(ctx, "get_current_stage_table") else None - ) - except Exception: - current_stage_table = None - try: - last_result_table = ( - ctx.get_last_result_table() - if hasattr(ctx, "get_last_result_table") else None - ) - except Exception: - last_result_table = None - - if current_stage_table is not None and current_stage_table is not pre_stage_table: - output_table = current_stage_table - elif last_result_table is not None and last_result_table is not pre_last_result_table: - output_table = last_result_table - - # Update piped_result for next stage from emitted items - stage_emits = list(stage_ctx.emits) - if stage_emits: - piped_result = stage_emits if len(stage_emits) > 1 else stage_emits[0] - else: - piped_result = None - finally: - if progress_ui is not None and pipe_idx is not None: - progress_ui.finish_pipe(pipe_idx) - if output_table is not None: - try: - from SYS.rich_display import stdout_console - - stdout_console().print() - stdout_console().print(output_table) - except Exception: - logger.exception("Failed to render output_table to stdout_console") - if session: - try: - session.close() - except Exception: - logger.exception("Failed to close pipeline stage session") - - except Exception as exc: - pipeline_status = "failed" - pipeline_error = f"{cmd_name}: {exc}" - debug(f"Error in stage {stage_index} ({cmd_name}): {exc}") - return - except Exception as exc: - pipeline_status = "failed" - pipeline_error = f"{type(exc).__name__}: {exc}" - print(f"[error] {type(exc).__name__}: {exc}\n") - finally: - # Stop Live progress and clear pipeline-level live progress - if progress_ui is not None: - try: - progress_ui.complete_all_pipes() - except Exception: - logger.exception("Failed to complete all pipe UI tasks in progress_ui.complete_all_pipes") - try: - progress_ui.stop() - except Exception: - logger.exception("Failed to stop progress_ui") - try: - from SYS import pipeline as _pipeline_ctx - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(None) - except Exception: - logger.exception("Failed to clear live_progress on pipeline context") - # Close pipeline session and log final status - try: - if pipeline_session and worker_manager: - pipeline_session.close(status=pipeline_status, error_msg=pipeline_error) - except Exception: - logger.exception("Failed to close pipeline session during finalization") - try: - if pipeline_session and worker_manager: - self._log_pipeline_event(worker_manager, pipeline_session.worker_id, - f"Pipeline {pipeline_status}: {pipeline_error or ''}") - except Exception: - logger.exception("Failed to log final pipeline status (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - try: - set_last_execution_result( - status=pipeline_status, - error=pipeline_error, - command_text=pipeline_text, - ) - except Exception: - logger.exception("Failed to record last execution result for pipeline") +__all__ = list(_state_all) + ["PipelineExecutor"] diff --git a/SYS/pipeline_executor.py b/SYS/pipeline_executor.py new file mode 100644 index 0000000..ccc6f54 --- /dev/null +++ b/SYS/pipeline_executor.py @@ -0,0 +1,2496 @@ +""" +Pipeline execution engine for cmdlet. +""" +from __future__ import annotations + +import sys +import time +import re +from datetime import datetime +from typing import Any, Dict, List, Optional, Sequence, Callable + +from SYS.cmdlet_catalog import import_cmd_module +from SYS.logger import debug +from SYS.pipeline_state import ( + HELP_EXAMPLE_SOURCE_COMMANDS, + _worker, + _cli_parsing, + _result_table, + _split_pipeline_tokens, + _emit_selection_debug_panel, + _is_selectable_table, +) + +import logging + +logger = logging.getLogger(__name__) + + +# SYS.rich_display deferred imports are handled inline where needed. +# SYS.background_notifier deferred imports are handled inline where needed. + + +class PipelineExecutor: + def __init__(self, *, config_loader: Optional[Any] = None) -> None: + self._config_loader = config_loader + self._toolbar_output: Optional[Callable[[str], None]] = None + + def _load_config(self) -> Dict[str, Any]: + try: + if self._config_loader is not None: + return self._config_loader.load() + except Exception: + logger.exception( + "Failed to use config_loader.load(); falling back to SYS.config.load_config" + ) + try: + from SYS.config import load_config + + return load_config() + except Exception: + return {} + + def set_toolbar_output(self, output: Optional[Callable[[str], None]]) -> None: + self._toolbar_output = output + + # ------------------------------------------------------------------- + # Static helpers + # ------------------------------------------------------------------- + + @staticmethod + def _split_stages(tokens: Sequence[str]) -> List[List[str]]: + stages: List[List[str]] = [] + current: List[str] = [] + for token in tokens: + if token == "|": + if current: + stages.append(current) + current = [] + else: + current.append(token) + if current: + stages.append(current) + return stages + + @staticmethod + def _stage_file_action(stage_tokens: Sequence[Any]) -> Optional[str]: + if not stage_tokens: + return None + + head = str(stage_tokens[0] or "").replace("_", "-").strip().lower() + if head in {"add-file", "download-file", "delete-file"}: + return head + if head != "file": + return None + args = {str(t).replace("_", "-").strip().lower() for t in stage_tokens[1:]} + if "-add" in args or "--add" in args: + return "add-file" + if "-download" in args or "--download" in args or "-dl" in args or "--dl" in args: + return "download-file" + if "-delete" in args or "--delete" in args or "-del" in args or "--del" in args: + return "delete-file" + return None + + @staticmethod + def _validate_download_file_relationship_order(stages: List[List[str]]) -> bool: + def _norm(name: str) -> str: + return str(name or "").replace("_", "-").strip().lower() + + def _file_action(stage_tokens: List[str]) -> Optional[str]: + if not stage_tokens: + return None + head = _norm(stage_tokens[0]) + if head in {"download-file", "add-file"}: + return head + if head != "file": + return None + args = {_norm(t) for t in stage_tokens[1:]} + if "-download" in args or "--download" in args or "-dl" in args or "--dl" in args: + return "download-file" + if "-add" in args or "--add" in args: + return "add-file" + return None + + names: List[str] = [] + for stage in stages or []: + if not stage: + continue + names.append(_norm(stage[0])) + + dl_idxs = [ + i for i, stage in enumerate(stages or []) if _file_action(stage or []) == "download-file" + ] + rel_idxs = [i for i, n in enumerate(names) if n == "add-relationship"] + add_file_idxs = [ + i for i, stage in enumerate(stages or []) if _file_action(stage or []) == "add-file" + ] + + if not dl_idxs or not rel_idxs: + return True + + for rel_i in rel_idxs: + dl_before = [d for d in dl_idxs if d < rel_i] + if not dl_before: + continue + dl_i = max(dl_before) + if not any(dl_i < a < rel_i for a in add_file_idxs): + print( + "Pipeline order error: when using download-file with add-relationship, " + "add-relationship must come after add-file (so items are stored and have store+hash).\n" + "Example: download-file <...> | add-file -instance | add-relationship\n" + ) + return False + + return True + + @staticmethod + def _try_clear_pipeline_stop(ctx: Any) -> None: + try: + if hasattr(ctx, "clear_pipeline_stop"): + ctx.clear_pipeline_stop() + except Exception: + logger.exception("Failed to clear pipeline stop via ctx.clear_pipeline_stop") + + @staticmethod + def _maybe_seed_current_stage_table(ctx: Any) -> None: + try: + if hasattr(ctx, "get_current_stage_table") and not ctx.get_current_stage_table(): + display_table = ( + ctx.get_display_table() if hasattr(ctx, "get_display_table") else None + ) + if display_table: + ctx.set_current_stage_table(display_table) + else: + last_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") + else None + ) + if last_table: + ctx.set_current_stage_table(last_table) + except Exception: + logger.exception("Failed to seed current_stage_table from display or last table") + + @staticmethod + def _maybe_apply_pending_pipeline_tail( + ctx: Any, stages: List[List[str]] + ) -> List[List[str]]: + try: + pending_tail = ( + ctx.get_pending_pipeline_tail() + if hasattr(ctx, "get_pending_pipeline_tail") + else [] + ) + pending_source = ( + ctx.get_pending_pipeline_source() + if hasattr(ctx, "get_pending_pipeline_source") + else None + ) + except Exception: + pending_tail = [] + pending_source = None + + try: + current_source = ( + ctx.get_current_stage_table_source_command() + if hasattr(ctx, "get_current_stage_table_source_command") + else None + ) + except Exception: + current_source = None + + try: + effective_source = current_source or ( + ctx.get_last_result_table_source_command() + if hasattr(ctx, "get_last_result_table_source_command") + else None + ) + except Exception: + effective_source = current_source + + selection_start = bool(stages and stages[0] and stages[0][0].startswith("@")) + + def _tail_is_suffix(existing: List[List[str]], tail: List[List[str]]) -> bool: + if not tail or not existing: + return False + if len(tail) > len(existing): + return False + return existing[-len(tail) :] == tail + + if pending_tail and selection_start: + if (pending_source is None) or ( + effective_source and pending_source == effective_source + ): + if not _tail_is_suffix(stages, pending_tail): + stages = list(stages) + list(pending_tail) + try: + if hasattr(ctx, "clear_pending_pipeline_tail"): + ctx.clear_pending_pipeline_tail() + except Exception: + logger.exception( + "Failed to clear pending pipeline tail after appending pending tail" + ) + else: + try: + if hasattr(ctx, "clear_pending_pipeline_tail"): + ctx.clear_pending_pipeline_tail() + except Exception: + logger.exception( + "Failed to clear pending pipeline tail (source mismatch branch)" + ) + elif pending_tail: + try: + if hasattr(ctx, "clear_pending_pipeline_tail"): + ctx.clear_pending_pipeline_tail() + except Exception: + logger.exception("Failed to clear stale pending pipeline tail") + return stages + + def _apply_quiet_background_flag(self, config: Any) -> Any: + if isinstance(config, dict): + try: + is_tty = bool(getattr(sys.stderr, "isatty", lambda: False)()) + except Exception: + is_tty = False + config["_quiet_background_output"] = not is_tty + return config + + @staticmethod + def _extract_first_stage_selection_tokens( + stages: List[List[str]], + ) -> tuple[List[List[str]], List[int], bool, bool]: + first_stage_tokens = stages[0] if stages else [] + first_stage_selection_indices: List[int] = [] + first_stage_had_extra_args = False + first_stage_select_all = False + + if first_stage_tokens: + new_first_stage: List[str] = [] + for token in first_stage_tokens: + if token.startswith("@"): + selection = _cli_parsing().SelectionSyntax.parse(token) + if selection is not None: + first_stage_selection_indices = [i - 1 for i in selection] + continue + if token == "@*": + first_stage_select_all = True + continue + new_first_stage.append(token) + + if new_first_stage: + stages = list(stages) + stages[0] = new_first_stage + if first_stage_selection_indices or first_stage_select_all: + first_stage_had_extra_args = True + elif first_stage_selection_indices or first_stage_select_all: + stages = list(stages) + stages.pop(0) + + return ( + stages, + first_stage_selection_indices, + first_stage_had_extra_args, + first_stage_select_all, + ) + + @staticmethod + def _apply_select_all_if_requested( + ctx: Any, indices: List[int], select_all: bool + ) -> List[int]: + if not select_all: + return indices + try: + last_items = ctx.get_last_result_items() + except Exception: + last_items = None + if last_items: + return list(range(len(last_items))) + return indices + + @staticmethod + def _maybe_run_class_selector( + ctx: Any, + config: Any, + selected_items: list, + *, + stage_is_last: bool, + source_command: Any = None, + prefer_detail_fallback: bool = False, + ) -> bool: + if not stage_is_last: + return False + + candidates: list[str] = [] + seen: set[str] = set() + current_table = None + table_meta = None + table_type = "" + + def _add(value: Any) -> None: + try: + text = str(value or "").strip().lower() + except Exception as exc: + logger.debug( + "Failed to normalize candidate value: %s", exc, exc_info=True + ) + return + if not text or text in seen: + return + seen.add(text) + candidates.append(text) + + try: + current_table = ctx.get_current_stage_table() or ctx.get_last_result_table() + _add( + current_table.table + if current_table and hasattr(current_table, "table") + else None + ) + if current_table and hasattr(current_table, "table"): + table_type = str(getattr(current_table, "table", "") or "").strip() + + try: + meta = ( + current_table.get_table_metadata() + if current_table is not None + and hasattr(current_table, "get_table_metadata") + else getattr(current_table, "table_metadata", None) + ) + except Exception: + meta = None + table_meta = meta if isinstance(meta, dict) else None + if isinstance(meta, dict): + _add(meta.get("plugin")) + except Exception: + logger.exception( + "Failed to inspect current_table/table metadata in _maybe_run_class_selector" + ) + + for item in selected_items or []: + if isinstance(item, dict): + _add(item.get("plugin")) + _add(item.get("store")) + _add(item.get("table")) + else: + _add(getattr(item, "plugin", None)) + _add(getattr(item, "store", None)) + _add(getattr(item, "table", None)) + + try: + from PluginCore.registry import get_plugin, is_known_plugin_name + except Exception: + get_plugin = None # type: ignore + is_known_plugin_name = None # type: ignore + + if is_known_plugin_name is not None: + try: + for key in list(candidates): + if not isinstance(key, str): + continue + if "." not in key: + continue + if is_known_plugin_name(key): + continue + prefix = str(key).split(".", 1)[0].strip().lower() + if prefix and is_known_plugin_name(prefix): + _add(prefix) + except Exception: + logger.exception( + "Failed while computing plugin prefix heuristics in _maybe_run_class_selector" + ) + + if get_plugin is not None: + for key in candidates: + try: + if is_known_plugin_name is not None and ( + not is_known_plugin_name(key) + ): + continue + except Exception: + logger.exception( + "is_known_plugin_name predicate failed for key %s; falling back", + key, + ) + try: + provider = get_plugin(key, config) + except Exception as exc: + logger.exception( + "Failed to load plugin '%s' during selector resolution: %s", + key, + exc, + ) + continue + selector = getattr(provider, "selector", None) + if selector is None: + continue + try: + handled = bool( + selector(selected_items, ctx=ctx, stage_is_last=True) + ) + except Exception as exc: + logger.exception( + "%s selector failed during selection: %s", key, exc + ) + return True + if handled: + return True + + if prefer_detail_fallback: + detail_renderer = getattr(provider, "show_selection_details", None) + if callable(detail_renderer): + try: + detail_handled = bool( + detail_renderer( + selected_items, + ctx=ctx, + stage_is_last=True, + source_command=str(source_command or ""), + table_type=table_type, + table_metadata=table_meta, + ) + ) + except Exception as exc: + logger.exception( + "%s detail fallback failed during selection: %s", + key, + exc, + ) + return True + if detail_handled: + return True + + store_keys: list[str] = [] + for item in selected_items or []: + if isinstance(item, dict): + v = item.get("store") + else: + v = getattr(item, "store", None) + name = str(v or "").strip() + if name: + store_keys.append(name) + + if store_keys: + try: + from PluginCore.backend_registry import BackendRegistry + + backend_registry = BackendRegistry(config, suppress_debug=True) + _backend_names = list(backend_registry.list_backends() or []) + _backend_by_lower = { + str(n).lower(): str(n) for n in _backend_names if str(n).strip() + } + for name in store_keys: + resolved_name = name + if not backend_registry.is_available(resolved_name): + resolved_name = _backend_by_lower.get( + str(name).lower(), name + ) + if not backend_registry.is_available(resolved_name): + continue + backend = backend_registry[resolved_name] + selector = getattr(backend, "selector", None) + if selector is None: + continue + handled = bool( + selector(selected_items, ctx=ctx, stage_is_last=True) + ) + if handled: + return True + except Exception: + logger.exception( + "Failed while running store-based selector logic in _maybe_run_class_selector" + ) + + return False + + @staticmethod + def _maybe_expand_plugin_selection( + selected_items: List[Any], + *, + ctx: Any, + config: Dict[str, Any], + stage_table: Any, + ) -> Optional[List[Any]]: + candidates: list[str] = [] + + def _add(value: Any) -> None: + text = str(value or "").strip().lower() + if text and text not in candidates: + candidates.append(text) + + table_type = None + try: + table_type = ( + stage_table.table + if stage_table is not None and hasattr(stage_table, "table") + else None + ) + except Exception: + table_type = None + _add(table_type) + + try: + meta = ( + stage_table.get_table_metadata() + if stage_table is not None and hasattr(stage_table, "get_table_metadata") + else getattr(stage_table, "table_metadata", None) + ) + except Exception: + meta = None + if isinstance(meta, dict): + _add(meta.get("plugin")) + + for item in selected_items or []: + if isinstance(item, dict): + _add(item.get("plugin")) + _add(item.get("table")) + _add(item.get("source")) + else: + _add(getattr(item, "plugin", None)) + _add(getattr(item, "table", None)) + _add(getattr(item, "source", None)) + + try: + from PluginCore.registry import get_plugin, is_known_plugin_name + except Exception: + return None + + for key in list(candidates): + if "." in key: + prefix = str(key).split(".", 1)[0].strip().lower() + if prefix and prefix not in candidates: + candidates.append(prefix) + + for key in candidates: + try: + if not is_known_plugin_name(key): + continue + except Exception: + continue + try: + plugin = get_plugin(key, config) + except Exception: + continue + if plugin is None: + continue + expand = getattr(plugin, "expand_selection", None) + if not callable(expand): + continue + try: + expanded = expand( + selected_items, + ctx=ctx, + stage_is_last=False, + table_type=str(table_type or ""), + ) + except Exception: + logger.exception("%s expand_selection failed", key) + return None + if expanded: + return list(expanded) + return None + + @staticmethod + def _summarize_stage_text(stage_tokens: Sequence[str], limit: int = 140) -> str: + combined = " ".join(str(tok) for tok in stage_tokens if tok is not None).strip() + if not combined: + return "" + normalized = re.sub(r"\s+", " ", combined) + if len(normalized) <= limit: + return normalized + return normalized[: limit - 3].rstrip() + "..." + + @staticmethod + def _log_pipeline_event( + worker_manager: Any, + worker_id: Optional[str], + message: str, + ) -> None: + if not worker_manager or not worker_id or not message: + return + try: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + except Exception: + timestamp = "" + if timestamp: + text = f"{timestamp} - PIPELINE - {message}" + else: + text = f"PIPELINE - {message}" + try: + worker_manager.append_stdout(worker_id, text + "\n", channel="log") + except Exception: + logger.exception( + "Failed to append pipeline event to worker stdout for %s", worker_id + ) + + @staticmethod + def _maybe_open_url_selection( + current_table: Any, + selected_items: list, + *, + stage_is_last: bool, + ) -> bool: + if not stage_is_last: + return False + if not selected_items or len(selected_items) != 1: + return False + + table_type = "" + source_cmd = "" + try: + table_type = ( + str(getattr(current_table, "table", "") or "").strip().lower() + ) + except Exception: + table_type = "" + try: + source_cmd = ( + str(getattr(current_table, "source_command", "") or "") + .strip() + .replace("_", "-") + .lower() + ) + except Exception: + source_cmd = "" + + if table_type != "url" and source_cmd != "get-url": + return False + + item = selected_items[0] + url = None + try: + from SYS.field_access import get_field + + url = get_field(item, "url") + except Exception: + try: + url = ( + item.get("url") + if isinstance(item, dict) + else getattr(item, "url", None) + ) + except Exception: + url = None + + url_text = str(url or "").strip() + if not url_text: + return False + + try: + import webbrowser + + webbrowser.open(url_text, new=2) + return True + except Exception: + return False + + # ------------------------------------------------------------------- + # Instance helpers + # ------------------------------------------------------------------- + + def _maybe_enable_background_notifier( + self, + worker_manager: Any, + config: Any, + pipeline_session: Any, + ) -> None: + if not (pipeline_session and worker_manager and isinstance(config, dict)): + return + + session_worker_ids = config.get("_session_worker_ids") + if not session_worker_ids: + return + + try: + output_fn = self._toolbar_output + quiet_mode = bool(config.get("_quiet_background_output")) + terminal_only = quiet_mode and not output_fn + kwargs: Dict[str, Any] = { + "session_worker_ids": session_worker_ids, + "only_terminal_updates": terminal_only, + "overlay_mode": bool(output_fn), + } + if output_fn: + kwargs["output"] = output_fn + from SYS.background_notifier import ensure_background_notifier + + ensure_background_notifier(worker_manager, **kwargs) + except Exception: + logger.exception( + "Failed to enable background notifier for session_worker_ids=%r", + session_worker_ids, + ) + + @staticmethod + def _get_raw_stage_texts(ctx: Any) -> List[str]: + raw_stage_texts: List[str] = [] + try: + if hasattr(ctx, "get_current_command_stages"): + raw_stage_texts = ctx.get_current_command_stages() or [] + except Exception: + raw_stage_texts = [] + return raw_stage_texts + + # ------------------------------------------------------------------- + # Selection expansion + # ------------------------------------------------------------------- + + def _maybe_apply_initial_selection( + self, + ctx: Any, + config: Any, + stages: List[List[str]], + *, + selection_indices: List[int], + first_stage_had_extra_args: bool, + worker_manager: Any, + pipeline_session: Any, + ) -> tuple[bool, Any]: + if not selection_indices: + return True, None + + # ------------------------------------------------------------------ + # PHASE 1: Synchronize current stage table with display table + # ------------------------------------------------------------------ + display_table = None + try: + display_table = ( + ctx.get_display_table() if hasattr(ctx, "get_display_table") else None + ) + except Exception: + display_table = None + + current_stage_table = None + try: + current_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") + else None + ) + except Exception: + current_stage_table = None + + try: + if display_table is not None and hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(display_table) + elif current_stage_table is None and hasattr(ctx, "set_current_stage_table"): + last_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") + else None + ) + if last_table is not None: + ctx.set_current_stage_table(last_table) + except Exception: + logger.exception( + "Failed to sync current_stage_table from display/last table in _maybe_apply_initial_selection" + ) + + # ------------------------------------------------------------------ + # Helper functions for row action/args discovery + # ------------------------------------------------------------------ + def _get_row_action( + idx: int, items_cache: Optional[List[Any]] = None + ) -> Optional[List[str]]: + try: + action = ctx.get_current_stage_table_row_selection_action(idx) + if action: + return [str(x) for x in action if x is not None] + except Exception: + pass + + if items_cache is None: + try: + items_cache = ctx.get_last_result_items() or [] + except Exception: + items_cache = [] + + if 0 <= idx < len(items_cache): + item = items_cache[idx] + if isinstance(item, dict): + candidate = item.get("_selection_action") + if isinstance(candidate, (list, tuple)): + return [str(x) for x in candidate if x is not None] + return None + + def _get_row_args( + idx: int, items_cache: Optional[List[Any]] = None + ) -> Optional[List[str]]: + try: + args = ctx.get_current_stage_table_row_selection_args(idx) + if args: + return [str(x) for x in args if x is not None] + except Exception: + pass + + if items_cache is None: + try: + items_cache = ctx.get_last_result_items() or [] + except Exception: + items_cache = [] + + if 0 <= idx < len(items_cache): + item = items_cache[idx] + if isinstance(item, dict): + candidate = item.get("_selection_args") + if isinstance(candidate, (list, tuple)): + return [str(x) for x in candidate if x is not None] + return None + + def _norm_cmd_name(value: Any) -> str: + return str(value or "").replace("_", "-").strip().lower() + + # ------------------------------------------------------------------ + # PHASE 2: Parse source command and table metadata + # ------------------------------------------------------------------ + source_cmd = None + source_args_raw = None + try: + source_cmd = ctx.get_current_stage_table_source_command() + source_args_raw = ctx.get_current_stage_table_source_args() + except Exception: + source_cmd = None + source_args_raw = None + + if isinstance(source_args_raw, str): + source_args: List[str] = [source_args_raw] + elif isinstance(source_args_raw, list): + source_args = [str(x) for x in source_args_raw if x is not None] + else: + source_args = [] + + current_table = None + try: + current_table = ctx.get_current_stage_table() + except Exception: + current_table = None + table_type = ( + current_table.table + if current_table and hasattr(current_table, "table") + else None + ) + + # ------------------------------------------------------------------ + # PHASE 3: Handle command expansion for @N syntax + # ------------------------------------------------------------------ + command_expanded = False + example_selector_triggered = False + normalized_source_cmd = ( + str(source_cmd or "").replace("_", "-").strip().lower() + ) + prefer_row_action = False + preferred_row_action = None + + if normalized_source_cmd in HELP_EXAMPLE_SOURCE_COMMANDS and selection_indices: + try: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + except Exception: + row_args = None + tokens: List[str] = [] + if isinstance(row_args, list) and row_args: + tokens = [str(x) for x in row_args if x is not None] + if tokens: + stage_groups = _split_pipeline_tokens(tokens) + if stage_groups: + for stage in reversed(stage_groups): + stages.insert(0, stage) + selection_indices = [] + command_expanded = True + example_selector_triggered = True + + if not example_selector_triggered: + if table_type in {"youtube", "soulseek"}: + command_expanded = False + elif source_cmd == "search-file" and source_args and "youtube" in source_args: + command_expanded = False + else: + selected_row_args: List[str] = [] + skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0 + if len(selection_indices) == 1 and not stages: + try: + row_action = _get_row_action(selection_indices[0]) + except Exception: + row_action = None + if row_action: + stages.insert(0, list(row_action)) + return True, None + + if source_cmd and not skip_pipe_expansion and not prefer_row_action: + src = str(source_cmd).replace("_", "-").strip().lower() + + if src == "add-file" and selection_indices: + row_args_list: List[List[str]] = [] + for idx in selection_indices: + try: + row_args = ctx.get_current_stage_table_row_selection_args( + idx + ) + except Exception: + row_args = None + if isinstance(row_args, list) and row_args: + row_args_list.append( + [str(x) for x in row_args if x is not None] + ) + + paths: List[str] = [] + can_merge = bool(row_args_list) and ( + len(row_args_list) == len(selection_indices) + ) + if can_merge: + for ra in row_args_list: + if len(ra) == 1: + p = str(ra[0]).strip() + if p: + paths.append(p) + else: + can_merge = False + break + + if can_merge and paths: + selected_row_args.append(",".join(paths)) + elif len(selection_indices) == 1 and row_args_list: + selected_row_args.extend(row_args_list[0]) + else: + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + if row_args: + selected_row_args.extend(row_args) + + if selected_row_args and not stages: + if isinstance(source_cmd, list): + cmd_list: List[str] = [ + str(x) for x in source_cmd if x is not None + ] + elif isinstance(source_cmd, str): + cmd_list = [source_cmd] + else: + cmd_list = [] + + expanded_stage: List[str] = ( + cmd_list + selected_row_args + source_args + ) + stages.insert(0, expanded_stage) + + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", + ) + except Exception: + logger.exception( + "Failed to record pipeline log step for @N expansion (pipeline_session=%r)", + getattr(pipeline_session, "worker_id", None), + ) + elif selected_row_args and stages: + pass + + stage_table = None + try: + stage_table = ctx.get_current_stage_table() + except Exception: + stage_table = None + + display_table = None + try: + display_table = ( + ctx.get_display_table() + if hasattr(ctx, "get_display_table") + else None + ) + except Exception: + display_table = None + + if not stage_table and display_table is not None: + stage_table = display_table + if not stage_table: + try: + stage_table = ctx.get_last_result_table() + except Exception: + stage_table = None + + # ---------------------------------------------------------------- + # PHASE 4: Retrieve and filter items from current result set + # ---------------------------------------------------------------- + try: + items_list = ctx.get_last_result_items() or [] + except Exception as exc: + debug(f"@N: Exception getting items_list: {exc}") + items_list = [] + resolved_items = items_list if items_list else [] + if items_list: + filtered = [ + resolved_items[i] + for i in selection_indices + if 0 <= i < len(resolved_items) + ] + if selection_indices: + if len(selection_indices) == 1: + selection_label = f"@{selection_indices[0] + 1}" + else: + selection_label = ( + "@{" + + ",".join(str(idx + 1) for idx in selection_indices) + + "}" + ) + else: + selection_label = "@selection" + _emit_selection_debug_panel( + selection_token=selection_label, + selection_indices=selection_indices, + item_count=len(items_list), + filtered_count=len(filtered), + stage_table_present=(stage_table is not None), + display_table_present=(display_table is not None), + stage_is_last=(not stages), + row_action=preferred_row_action, + downstream_stages=stages, + mode=("row_action" if preferred_row_action else "selection"), + ) + if not filtered: + print("No items matched selection in pipeline\n") + return False, None + + if stages: + expanded = PipelineExecutor._maybe_expand_plugin_selection( + filtered, + ctx=ctx, + config=config, + stage_table=stage_table, + ) + if expanded: + filtered = expanded + + if PipelineExecutor._maybe_run_class_selector( + ctx, + config, + filtered, + stage_is_last=(not stages), + source_command=source_cmd, + prefer_detail_fallback=bool( + prefer_row_action + and not stages + and len(selection_indices) == 1 + ), + ): + return False, None + + from SYS.pipe_object import coerce_to_pipe_object + + filtered_pipe_objs = [ + coerce_to_pipe_object(item) for item in filtered + ] + piped_result = ( + filtered_pipe_objs + if len(filtered_pipe_objs) > 1 + else filtered_pipe_objs[0] + ) + + if pipeline_session and worker_manager: + try: + selection_parts = [f"@{i+1}" for i in selection_indices] + worker_manager.log_step( + pipeline_session.worker_id, + f"Applied @N selection {' | '.join(selection_parts)}", + ) + except Exception: + logger.exception( + "Failed to record Applied @N selection log step (pipeline_session=%r)", + getattr(pipeline_session, "worker_id", None), + ) + + try: + current_table = ctx.get_current_stage_table() + if current_table is None and hasattr(ctx, "get_display_table"): + current_table = ctx.get_display_table() + if current_table is None: + current_table = ctx.get_last_result_table() + except Exception: + logger.exception( + "Failed to determine current_table for selection auto-insert; defaulting to None" + ) + current_table = None + table_type_hint = None + try: + raw_table_type = ( + stage_table.table + if stage_table is not None and hasattr(stage_table, "table") + else None + ) + if isinstance(raw_table_type, str) and raw_table_type.strip(): + table_type_hint = raw_table_type + except Exception: + table_type_hint = None + table_type = None + try: + if isinstance(table_type_hint, str) and table_type_hint.strip(): + table_type = table_type_hint + else: + table_type = ( + current_table.table + if current_table and hasattr(current_table, "table") + else None + ) + except Exception: + logger.exception( + "Failed to compute table_type from current_table; using fallback attribute access" + ) + table_type = ( + current_table.table + if current_table and hasattr(current_table, "table") + else None + ) + + def _norm_cmd(name: Any) -> str: + return str(name or "").replace("_", "-").strip().lower() + + auto_stage = None + if isinstance(table_type, str) and table_type: + try: + from PluginCore.registry import ( + selection_auto_stage_for_table, + ) + + auto_stage = selection_auto_stage_for_table(table_type) + except Exception: + auto_stage = None + + source_cmd_for_selection = None + source_args_for_selection: List[str] = [] + try: + source_cmd_for_selection = ( + ctx.get_current_stage_table_source_command() + or ctx.get_last_result_table_source_command() + ) + source_args_for_selection = ( + ctx.get_current_stage_table_source_args() + or ctx.get_last_result_table_source_args() + or [] + ) + except Exception: + source_cmd_for_selection = None + source_args_for_selection = [] + + if not stages and selection_indices and source_cmd_for_selection: + src_norm = _norm_cmd_name(source_cmd_for_selection) + if src_norm in {".worker", "worker", "workers"}: + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = _get_row_args(idx, items_list) + if row_args: + stages.append( + [str(source_cmd_for_selection)] + + row_args + + [ + str(x) + for x in source_args_for_selection + if x is not None + ] + ) + + def _apply_row_action_to_stage(stage_idx: int) -> bool: + if not selection_indices or len(selection_indices) != 1: + return False + row_action = _get_row_action(selection_indices[0], items_list) + if not row_action: + return False + if 0 <= stage_idx < len(stages): + stages[stage_idx] = row_action + return True + return False + + # ---------------------------------------------------------------- + # PHASE 5: Auto-insert stages based on table type and user selection + # ---------------------------------------------------------------- + if not stages: + if isinstance(table_type, str) and table_type.startswith( + "metadata." + ): + print("Auto-applying metadata selection via metadata -get") + stages.append(["metadata", "-get"]) + elif auto_stage: + try: + print(f"Auto-running selection via {auto_stage[0]}") + except Exception: + logger.exception( + "Failed to print auto-run selection message for %s", + auto_stage[0], + ) + stages.append(list(auto_stage)) + debug( + f"Inserted auto stage before row action: {stages[-1]}" + ) + + if selection_indices: + try: + if not _apply_row_action_to_stage(len(stages) - 1): + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = _get_row_args(idx, items_list) + if row_args: + inserted = stages[-1] + if inserted: + cmd = inserted[0] + tail = [ + str(x) for x in inserted[1:] + ] + stages[-1] = [cmd] + row_args + tail + except Exception: + logger.exception( + "Failed to attach selection args to auto-inserted stage" + ) + + if ( + not stages + and selection_indices + and len(selection_indices) == 1 + ): + row_action = _get_row_action(selection_indices[0], items_list) + if row_action: + stages.append(row_action) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + f"@N applied row action -> {' '.join(row_action)}", + ) + except Exception: + logger.exception( + "Failed to record pipeline log step for applied row action (pipeline_session=%r)", + getattr( + pipeline_session, "worker_id", None + ), + ) + else: + first_cmd = stages[0][0] if stages and stages[0] else None + first_cmd_norm = _norm_cmd_name(first_cmd) + + inserted_provider_download = False + if ( + PipelineExecutor._stage_file_action(stages[0]) + == "add-file" + ): + if len(selection_indices) == 1: + row_action = _get_row_action( + selection_indices[0], items_list + ) + if ( + row_action + and PipelineExecutor._stage_file_action(row_action) + == "download-file" + ): + stages.insert( + 0, + [str(x) for x in row_action if x is not None], + ) + inserted_provider_download = True + debug( + "Auto-inserting row download-file action before add-file" + ) + + if (not inserted_provider_download) and len( + selection_indices + ) > 1: + try: + has_download_row_action = False + for idx in selection_indices: + row_action = _get_row_action( + idx, items_list + ) + if ( + row_action + and PipelineExecutor._stage_file_action( + row_action + ) + == "download-file" + ): + has_download_row_action = True + break + if has_download_row_action: + stages.insert(0, ["file", "-download"]) + inserted_provider_download = True + debug( + "Auto-inserting download-file before add-file for provider selection" + ) + except Exception: + pass + + if ( + isinstance(table_type, str) + and table_type.startswith("metadata.") + and first_cmd + not in ( + "metadata", + "tag", + ".pipe", + ".mpv", + ) + ): + print( + "Auto-inserting metadata -get after metadata selection" + ) + stages.insert(0, ["metadata", "-get"]) + elif auto_stage: + first_cmd_norm = _norm_cmd_name( + stages[0][0] if stages and stages[0] else None + ) + auto_cmd_norm = _norm_cmd_name(auto_stage[0]) + if first_cmd_norm not in ( + auto_cmd_norm, + ".pipe", + ".mpv", + ): + debug( + f"Auto-inserting {auto_cmd_norm} after selection" + ) + stages.insert(0, list(auto_stage)) + debug( + f"Inserted auto stage before existing pipeline: {stages[0]}" + ) + + if selection_indices: + try: + if not _apply_row_action_to_stage(0): + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = _get_row_args( + idx, items_list + ) + if row_args: + inserted = stages[0] + if inserted: + cmd = inserted[0] + tail = [ + str(x) + for x in inserted[1:] + ] + stages[0] = ( + [cmd] + row_args + tail + ) + except Exception: + logger.exception( + "Failed to attach selection args to inserted auto stage (alternate branch)" + ) + + if (not stages) and selection_indices and len(selection_indices) == 1: + try: + selected_item = filtered[0] if filtered else None + if selected_item is not None and not isinstance( + selected_item, dict + ): + to_dict = getattr(selected_item, "to_dict", None) + if callable(to_dict): + selected_item = to_dict() + if isinstance(selected_item, dict): + from SYS.rich_display import ( + render_item_details_panel, + ) + + render_item_details_panel(selected_item) + try: + ctx.set_last_result_items_only([selected_item]) + except Exception: + pass + except Exception: + logger.exception( + "Failed to render selection-only item details" + ) + return True, piped_result + else: + debug("@N: No items to select from (items_list empty)") + print("No previous results to select from\n") + return False, None + + return True, None + + # ------------------------------------------------------------------- + # Live progress + # ------------------------------------------------------------------- + + @staticmethod + def _maybe_start_live_progress( + config: Any, stages: List[List[str]] + ) -> tuple[Any, Dict[int, int]]: + progress_ui = None + pipe_index_by_stage: Dict[int, int] = {} + + try: + quiet_mode = ( + bool(config.get("_quiet_background_output")) + if isinstance(config, dict) + else False + ) + except Exception: + quiet_mode = False + + try: + import sys as _sys + + if (not quiet_mode) and bool( + getattr(_sys.stderr, "isatty", lambda: False)() + ): + from SYS.models import PipelineLiveProgress + + pipe_stage_indices: List[int] = [] + pipe_labels: List[str] = [] + for idx, stage_tokens in enumerate(stages): + if not stage_tokens: + continue + name = str(stage_tokens[0]).replace("_", "-").lower() + if name == "@" or name.startswith("@"): + continue + + if ( + PipelineExecutor._stage_file_action(stage_tokens) + == "add-file" + or name in {"add_file"} + ): + try: + from pathlib import Path as _Path + + toks = list(stage_tokens[1:]) + i = 0 + while i < len(toks): + t = str(toks[i]) + low = t.lower().strip() + if low in {"-path", "--path", "-p"} and i + 1 < len( + toks + ): + nxt = str(toks[i + 1]) + if nxt and ("," not in nxt): + p = _Path(nxt) + if p.exists() and p.is_dir(): + name = "" + break + i += 2 + continue + i += 1 + except Exception: + logger.exception( + "Failed to inspect add-file stage tokens for potential directory; skipping Live progress" + ) + if not name: + continue + if name in {"get-relationship", "get-rel"}: + continue + if name in {"get-metadata", "meta"}: + continue + if name in {".pipe", ".mpv"}: + continue + if name == ".matrix": + continue + if ( + PipelineExecutor._stage_file_action(stage_tokens) + == "delete-file" + or name in {"del-file"} + ): + continue + pipe_stage_indices.append(idx) + pipe_labels.append(name) + + if pipe_labels: + progress_ui = PipelineLiveProgress(pipe_labels, enabled=True) + progress_ui.start() + try: + from SYS import pipeline as _pipeline_ctx + + if hasattr(_pipeline_ctx, "set_live_progress"): + _pipeline_ctx.set_live_progress(progress_ui) + if hasattr(_pipeline_ctx, "get_progress_event_callback"): + progress_cb = _pipeline_ctx.get_progress_event_callback() + if callable(progress_cb) and hasattr( + progress_ui, "set_event_callback" + ): + progress_ui.set_event_callback(progress_cb) + except Exception: + logger.exception( + "Failed to register PipelineLiveProgress with pipeline context" + ) + pipe_index_by_stage = { + stage_idx: pipe_idx + for pipe_idx, stage_idx in enumerate(pipe_stage_indices) + } + except Exception: + progress_ui = None + pipe_index_by_stage = {} + + return progress_ui, pipe_index_by_stage + + # ------------------------------------------------------------------- + # Main execution entry point + # ------------------------------------------------------------------- + + def execute_tokens(self, tokens: List[str]) -> None: + from cmdlet import REGISTRY + + from SYS import pipeline_state as ctx + + try: + try: + from SYS.logger import debug_panel + + debug_panel( + "Pipeline execution", + [ + ("command", " ".join(str(tok) for tok in tokens)), + ("token_count", len(tokens)), + ], + ) + except Exception: + debug(f"execute_tokens: tokens={tokens}") + self._try_clear_pipeline_stop(ctx) + + try: + if hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(None) + except Exception: + logger.exception( + "Failed to clear current_stage_table in execute_tokens" + ) + + try: + ctx.store_value("preflight", {}) + except Exception: + logger.exception( + "Failed to set preflight cache in execute_tokens" + ) + + stages = self._split_stages(tokens) + if not stages: + print("Invalid pipeline syntax\n") + return + self._maybe_seed_current_stage_table(ctx) + stages = self._maybe_apply_pending_pipeline_tail(ctx, stages) + config = self._load_config() + config = self._apply_quiet_background_flag(config) + + ( + stages, + first_stage_selection_indices, + first_stage_had_extra_args, + first_stage_select_all, + ) = self._extract_first_stage_selection_tokens(stages) + first_stage_selection_indices = self._apply_select_all_if_requested( + ctx, first_stage_selection_indices, first_stage_select_all + ) + + piped_result: Any = None + worker_manager = _worker().WorkerManagerRegistry.ensure(config) + pipeline_text = " | ".join(" ".join(stage) for stage in stages) + pipeline_session = _worker().WorkerStages.begin_pipeline( + worker_manager, + pipeline_text=pipeline_text, + config=config, + ) + if pipeline_session and worker_manager: + self._log_pipeline_event( + worker_manager, + pipeline_session.worker_id, + f"Pipeline start: {pipeline_text or '(empty pipeline)'}", + ) + raw_stage_texts = self._get_raw_stage_texts(ctx) + self._maybe_enable_background_notifier( + worker_manager, config, pipeline_session + ) + + pipeline_status = "completed" + pipeline_error = "" + + progress_ui = None + pipe_index_by_stage: Dict[int, int] = {} + + ok, initial_piped = self._maybe_apply_initial_selection( + ctx, + config, + stages, + selection_indices=first_stage_selection_indices, + first_stage_had_extra_args=first_stage_had_extra_args, + worker_manager=worker_manager, + pipeline_session=pipeline_session, + ) + if not ok: + return + if initial_piped is not None: + piped_result = initial_piped + + if not self._validate_download_file_relationship_order(stages): + pipeline_status = "failed" + pipeline_error = "Invalid pipeline order" + return + + progress_ui, pipe_index_by_stage = self._maybe_start_live_progress( + config, stages + ) + + for stage_index, stage_tokens in enumerate(stages): + if not stage_tokens: + continue + + raw_stage_name = str(stage_tokens[0]) + cmd_name = raw_stage_name.replace("_", "-").lower() + stage_args = stage_tokens[1:] + + if cmd_name == "@": + try: + next_cmd = None + if stage_index + 1 < len(stages) and stages[ + stage_index + 1 + ]: + next_cmd = ( + str(stages[stage_index + 1][0]) + .replace("_", "-") + .strip() + .lower() + ) + + current_table = None + try: + current_table = ( + ctx.get_current_stage_table() + or ctx.get_last_result_table() + ) + except Exception: + current_table = None + + source_cmd = ( + str( + getattr(current_table, "source_command", "") or "" + ) + .replace("_", "-") + .strip() + .lower() + ) + is_get_tag_table = source_cmd == "metadata" + + if is_get_tag_table and next_cmd == "metadata": + subject = ctx.get_last_result_subject() + if subject is not None: + next_args = [ + str(x).replace("_", "-").strip().lower() + for x in ( + stages[stage_index + 1][1:] + if stage_index + 1 < len(stages) + else [] + ) + ] + if ( + "-add" not in next_args + and "--add" not in next_args + ): + pass + else: + piped_result = subject + try: + subject_items = ( + subject + if isinstance(subject, list) + else [subject] + ) + ctx.set_last_items(subject_items) + except Exception: + logger.exception( + "Failed to set last_items from tag subject during @ handling" + ) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + "@ used metadata table subject for metadata -add", + ) + except Exception: + logger.exception( + "Failed to record pipeline log step for '@ used metadata table subject for metadata -add' (pipeline_session=%r)", + getattr( + pipeline_session, + "worker_id", + None, + ), + ) + continue + except Exception: + logger.exception( + "Failed to evaluate tag @ subject special-case" + ) + + last_items = None + try: + last_items = ctx.get_last_result_items() + except Exception: + last_items = None + + if last_items: + from SYS.pipe_object import coerce_to_pipe_object + + try: + pipe_items = [ + coerce_to_pipe_object(x) + for x in list(last_items) + ] + except Exception: + pipe_items = list(last_items) + piped_result = ( + pipe_items + if len(pipe_items) > 1 + else pipe_items[0] + ) + try: + ctx.set_last_items(pipe_items) + except Exception: + logger.exception( + "Failed to set last items after @ selection" + ) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + "@ used last result items", + ) + except Exception: + logger.exception( + "Failed to record pipeline log step for '@ used last result items' (pipeline_session=%r)", + getattr( + pipeline_session, "worker_id", None + ), + ) + continue + + subject = ctx.get_last_result_subject() + if subject is None: + print("No current result context available for '@'\n") + pipeline_status = "failed" + pipeline_error = "No result items/subject for @" + return + piped_result = subject + try: + subject_items = ( + subject + if isinstance(subject, list) + else [subject] + ) + ctx.set_last_items(subject_items) + except Exception: + logger.exception( + "Failed to set last_items from subject during @ handling" + ) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + "@ used current table subject", + ) + except Exception: + logger.exception( + "Failed to record pipeline log step for '@ used current table subject' (pipeline_session=%r)", + getattr( + pipeline_session, "worker_id", None + ), + ) + continue + + if cmd_name.startswith("@"): + selection_token = raw_stage_name + selection = _cli_parsing().SelectionSyntax.parse( + selection_token + ) + filter_spec = _cli_parsing().SelectionFilterSyntax.parse( + selection_token + ) + is_select_all = selection_token.strip() == "@*" + if ( + selection is None + and filter_spec is None + and not is_select_all + ): + print(f"Invalid selection: {selection_token}\n") + pipeline_status = "failed" + pipeline_error = f"Invalid selection {selection_token}" + return + + selected_indices: List[int] = [] + display_table = None + try: + display_table = ( + ctx.get_display_table() + if hasattr(ctx, "get_display_table") + else None + ) + except Exception: + display_table = None + + stage_table = ctx.get_current_stage_table() + try: + if display_table is not None and hasattr( + ctx, "set_current_stage_table" + ): + ctx.set_current_stage_table(display_table) + stage_table = display_table + except Exception: + logger.exception( + "Failed to set current_stage_table from display table during selection processing" + ) + + if not stage_table and display_table is not None: + stage_table = display_table + if not stage_table: + stage_table = ctx.get_last_result_table() + + try: + if hasattr(ctx, "debug_table_state"): + ctx.debug_table_state( + f"selection {selection_token}" + ) + except Exception: + logger.exception( + "Failed to debug_table_state during selection %s", + selection_token, + ) + + if display_table is not None and stage_table is display_table: + items_list = ctx.get_last_result_items() or [] + else: + if hasattr(ctx, "get_last_selectable_result_items"): + items_list = ( + ctx.get_last_selectable_result_items() or [] + ) + else: + items_list = ctx.get_last_result_items() or [] + + if is_select_all: + selected_indices = list(range(len(items_list))) + elif filter_spec is not None: + selected_indices = [ + i + for i, item in enumerate(items_list) + if _cli_parsing().SelectionFilterSyntax.matches( + item, filter_spec + ) + ] + else: + selected_indices = [ + i - 1 for i in selection + ] + + resolved_items = items_list if items_list else [] + filtered = [ + resolved_items[i] + for i in selected_indices + if 0 <= i < len(resolved_items) + ] + try: + debug( + f"Selection {selection_token} -> resolved_indices={selected_indices} filtered_count={len(filtered)}" + ) + if filtered: + sample = filtered[0] + if isinstance(sample, dict): + debug( + f"Selection sample: hash={sample.get('hash')} store={sample.get('store')} _selection_args={sample.get('_selection_args')} _selection_action={sample.get('_selection_action')}" + ) + else: + try: + debug( + f"Selection sample object: provider={getattr(sample, 'provider', None)} store={getattr(sample, 'store', None)}" + ) + except Exception: + logger.exception( + "Failed to debug selection sample object" + ) + except Exception: + logger.exception( + "Failed to produce selection debug sample for token %s", + selection_token, + ) + + if not filtered: + print("No items matched selection\n") + pipeline_status = "failed" + pipeline_error = "Empty selection" + return + + stage_is_last = stage_index + 1 >= len(stages) + if filter_spec is not None and stage_is_last: + try: + base_table = stage_table + if base_table is None: + base_table = ctx.get_last_result_table() + + if base_table is not None and hasattr( + base_table, "copy_with_title" + ): + new_table = base_table.copy_with_title( + getattr(base_table, "title", "") + or "Results" + ) + else: + new_table = _result_table()( + getattr(base_table, "title", "") + if base_table is not None + else "Results" + ) + + try: + if base_table is not None and getattr( + base_table, "table", None + ): + new_table.set_table( + str(getattr(base_table, "table")) + ) + except Exception: + logger.exception( + "Failed to set table on new_table for filter overlay" + ) + + try: + safe = str(selection_token)[1:].strip() + new_table.set_header_line( + f'filter: "{safe}"' + ) + except Exception: + logger.exception( + "Failed to set header line for filter overlay for token %s", + selection_token, + ) + + for item in filtered: + new_table.add_result(item) + + try: + ctx.set_last_result_table_overlay( + new_table, + items=list(filtered), + subject=ctx.get_last_result_subject(), + ) + except Exception: + logger.exception( + "Failed to set last_result_table_overlay for filter selection" + ) + + try: + from SYS.rich_display import ( + stdout_console, + ) + + stdout_console().print() + stdout_console().print(new_table) + except Exception: + logger.exception( + "Failed to render filter overlay to stdout_console" + ) + except Exception: + logger.exception( + "Failed while rendering filter overlay for selection %s", + selection_token, + ) + continue + + current_table = ( + ctx.get_current_stage_table() + or ctx.get_last_result_table() + ) + if (not is_select_all) and (len(filtered) == 1): + try: + PipelineExecutor._maybe_open_url_selection( + current_table, + filtered, + stage_is_last=( + stage_index + 1 >= len(stages) + ), + ) + except Exception: + logger.exception( + "Failed to open URL selection for table %s", + getattr(current_table, "table", None), + ) + + if PipelineExecutor._maybe_run_class_selector( + ctx, + config, + filtered, + stage_is_last=(stage_index + 1 >= len(stages)), + ): + return + + next_cmd: Optional[str] = None + next_args: List[str] = [] + try: + if stage_index + 1 < len(stages) and stages[ + stage_index + 1 + ]: + next_cmd = ( + str(stages[stage_index + 1][0]) + .replace("_", "-") + .lower() + ) + next_args = [ + str(x).replace("_", "-").strip().lower() + for x in stages[stage_index + 1][1:] + ] + except Exception: + logger.exception( + "Failed to determine next_cmd during selection expansion for stage_index %s", + stage_index, + ) + next_cmd = None + next_args = [] + + def _is_tag_row(obj: Any) -> bool: + try: + if ( + hasattr(obj, "__class__") + and obj.__class__.__name__ == "TagItem" + and hasattr(obj, "tag_name") + ): + return True + except Exception: + logger.exception( + "Failed to inspect TagItem object while checking _is_tag_row" + ) + try: + if isinstance(obj, dict) and obj.get("tag_name"): + return True + except Exception: + logger.exception( + "Failed to inspect dict tag_name while checking _is_tag_row" + ) + return False + + if ( + next_cmd == "tag" + and ("-delete" in next_args or "--delete" in next_args) + and len(filtered) > 1 + and all(_is_tag_row(x) for x in filtered) + ): + from SYS.field_access import get_field + + tags: List[str] = [] + first_hash = None + first_store = None + first_path = None + for item in filtered: + tag_name = get_field(item, "tag_name") + if tag_name: + tags.append(str(tag_name)) + if first_hash is None: + first_hash = get_field(item, "hash") + if first_store is None: + first_store = get_field(item, "store") + if first_path is None: + first_path = get_field( + item, "path" + ) or get_field(item, "target") + + if tags: + grouped = { + "table": "tag.selection", + "media_kind": "tag", + "hash": first_hash, + "store": first_store, + "path": first_path, + "tag": tags, + } + piped_result = grouped + continue + + from SYS.pipe_object import coerce_to_pipe_object + + filtered_pipe_objs = [ + coerce_to_pipe_object(item) for item in filtered + ] + piped_result = ( + filtered_pipe_objs + if len(filtered_pipe_objs) > 1 + else filtered_pipe_objs[0] + ) + + current_table = ( + ctx.get_current_stage_table() + or ctx.get_last_result_table() + ) + table_type = ( + current_table.table + if current_table + and hasattr(current_table, "table") + else None + ) + + def _norm_stage_cmd(name: Any) -> str: + return ( + str(name or "").replace("_", "-").strip().lower() + ) + + next_cmd = None + if stage_index + 1 < len(stages) and stages[ + stage_index + 1 + ]: + next_cmd = _norm_stage_cmd( + stages[stage_index + 1][0] + ) + + auto_stage = None + if isinstance(table_type, str) and table_type: + try: + from PluginCore.registry import ( + selection_auto_stage_for_table, + ) + + at_end = bool(stage_index + 1 >= len(stages)) + auto_stage = selection_auto_stage_for_table( + table_type, + stage_args if at_end else None, + ) + except Exception: + auto_stage = None + + if filter_spec is None: + if stage_index + 1 >= len(stages): + if auto_stage: + try: + print( + f"Auto-running selection via {auto_stage[0]}" + ) + except Exception: + logger.exception( + "Failed to print auto-run selection message for %s", + auto_stage[0], + ) + stages.append(list(auto_stage)) + else: + if auto_stage: + auto_cmd = _norm_stage_cmd(auto_stage[0]) + if next_cmd not in ( + auto_cmd, + ".pipe", + ".mpv", + ): + debug( + f"Auto-inserting {auto_cmd} after selection" + ) + stages.insert( + stage_index + 1, list(auto_stage) + ) + continue + + cmd_fn = REGISTRY.get(cmd_name) + try: + mod = import_cmd_module(cmd_name, reload_loaded=True) + data = getattr(mod, "CMDLET", None) if mod else None + if ( + data + and hasattr(data, "exec") + and callable(getattr(data, "exec")) + ): + from SYS.cmdlet_spec import ( + collect_registered_cmdlet_names, + ) + + run_fn = getattr(data, "exec") + for registered_name in collect_registered_cmdlet_names( + data, fallback_name=cmd_name + ): + REGISTRY[registered_name] = run_fn + cmd_fn = run_fn + except Exception: + pass + + if not cmd_fn: + try: + mod = import_cmd_module(cmd_name) + data = getattr(mod, "CMDLET", None) if mod else None + if ( + data + and hasattr(data, "exec") + and callable(getattr(data, "exec")) + ): + run_fn = getattr(data, "exec") + REGISTRY[cmd_name] = run_fn + cmd_fn = run_fn + except Exception: + cmd_fn = None + + if not cmd_fn: + print(f"Unknown command: {cmd_name}\n") + pipeline_status = "failed" + pipeline_error = f"Unknown command: {cmd_name}" + return + + try: + from SYS.models import PipelineStageContext + + pipe_idx = pipe_index_by_stage.get(stage_index) + + output_table: Optional[Any] = None + pre_stage_table: Optional[Any] = None + pre_last_result_table: Optional[Any] = None + session = _worker().WorkerStages.begin_stage( + worker_manager, + cmd_name=cmd_name, + stage_tokens=stage_tokens, + config=config, + command_text=pipeline_text + if pipeline_text + else " ".join(stage_tokens), + ) + try: + stage_ctx = PipelineStageContext( + stage_index=stage_index, + total_stages=len(stages), + pipe_index=pipe_idx, + worker_id=session.worker_id + if session + else None, + on_emit=( + lambda x: progress_ui.on_emit(pipe_idx, x) + ) + if progress_ui is not None + and pipe_idx is not None + else None, + ) + + ctx.set_stage_context(stage_ctx) + ctx.set_current_cmdlet_name(cmd_name) + ctx.set_current_stage_text(" ".join(stage_tokens)) + ctx.clear_emits() + + if progress_ui is not None and pipe_idx is not None: + progress_ui.begin_pipe(pipe_idx, total_items=1) + + try: + pre_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") + else None + ) + except Exception: + pre_stage_table = None + try: + pre_last_result_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") + else None + ) + except Exception: + pre_last_result_table = None + + ret_code = cmd_fn( + piped_result, stage_args, config + ) + if ret_code is not None: + try: + normalized_ret = int(ret_code) + except Exception: + normalized_ret = 0 + if normalized_ret != 0: + pipeline_status = "failed" + pipeline_error = f"Stage '{cmd_name}' failed with exit code {normalized_ret}" + return + + if stage_index + 1 < len(stages): + try: + post_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") + else None + ) + except Exception: + post_stage_table = None + if ( + post_stage_table is not None + and post_stage_table is not pre_stage_table + ): + has_emits = bool(stage_ctx.emits) + if not has_emits: + tail = stages[stage_index + 1 :] + source = None + try: + raw_source = getattr( + post_stage_table, + "source_command", + None, + ) + if raw_source: + source = ( + str(raw_source) + .replace("_", "-") + .strip() + .lower() + ) + except Exception: + source = None + try: + ctx.set_pending_pipeline_tail( + tail, source + ) + except Exception: + pass + if post_stage_table is not None: + try: + from SYS.rich_display import ( + stdout_console, + ) + + stdout_console().print() + stdout_console().print( + post_stage_table + ) + except Exception: + pass + logger.info( + "Pipeline paused for selection — use @N to continue" + ) + pipeline_status = "paused_selection" + return + + output_table = None + if stage_index + 1 >= len(stages): + try: + output_table = ( + ctx.get_display_table() + if hasattr(ctx, "get_display_table") + else None + ) + except Exception: + output_table = None + + if output_table is None: + current_stage_table = None + last_result_table = None + try: + current_stage_table = ( + ctx.get_current_stage_table() + if hasattr( + ctx, "get_current_stage_table" + ) + else None + ) + except Exception: + current_stage_table = None + try: + last_result_table = ( + ctx.get_last_result_table() + if hasattr( + ctx, "get_last_result_table" + ) + else None + ) + except Exception: + last_result_table = None + + if ( + current_stage_table is not None + and current_stage_table + is not pre_stage_table + ): + output_table = current_stage_table + elif ( + last_result_table is not None + and last_result_table + is not pre_last_result_table + ): + output_table = last_result_table + + stage_emits = list(stage_ctx.emits) + if stage_emits: + piped_result = ( + stage_emits + if len(stage_emits) > 1 + else stage_emits[0] + ) + else: + piped_result = None + finally: + if progress_ui is not None and pipe_idx is not None: + try: + progress_ui.finish_pipe(pipe_idx) + except Exception: + logger.exception( + "Failed to finish pipe in progress_ui" + ) + if output_table is not None: + try: + from SYS.rich_display import ( + stdout_console, + ) + + stdout_console().print() + stdout_console().print(output_table) + except Exception: + logger.exception( + "Failed to render output_table to stdout_console" + ) + if session: + try: + session.close() + except Exception: + logger.exception( + "Failed to close pipeline stage session" + ) + + except Exception as exc: + pipeline_status = "failed" + pipeline_error = f"{cmd_name}: {exc}" + debug( + f"Error in stage {stage_index} ({cmd_name}): {exc}" + ) + return + except Exception as exc: + pipeline_status = "failed" + pipeline_error = f"{type(exc).__name__}: {exc}" + print(f"[error] {type(exc).__name__}: {exc}\n") + finally: + if progress_ui is not None: + try: + progress_ui.complete_all_pipes() + except Exception: + logger.exception( + "Failed to complete all pipe UI tasks in progress_ui.complete_all_pipes" + ) + try: + progress_ui.stop() + except Exception: + logger.exception("Failed to stop progress_ui") + try: + from SYS import pipeline as _pipeline_ctx + + if hasattr(_pipeline_ctx, "set_live_progress"): + _pipeline_ctx.set_live_progress(None) + except Exception: + logger.exception( + "Failed to clear live_progress on pipeline context" + ) + try: + if pipeline_session and worker_manager: + pipeline_session.close( + status=pipeline_status, error_msg=pipeline_error + ) + except Exception: + logger.exception( + "Failed to close pipeline session during finalization" + ) + try: + if pipeline_session and worker_manager: + self._log_pipeline_event( + worker_manager, + pipeline_session.worker_id, + f"Pipeline {pipeline_status}: {pipeline_error or ''}", + ) + except Exception: + logger.exception( + "Failed to log final pipeline status (pipeline_session=%r)", + getattr(pipeline_session, "worker_id", None), + ) + try: + from SYS.pipeline_state import set_last_execution_result + + set_last_execution_result( + status=pipeline_status, + error=pipeline_error, + command_text=pipeline_text, + ) + except Exception: + logger.exception( + "Failed to record last execution result for pipeline" + ) + + +__all__ = ["PipelineExecutor"] diff --git a/SYS/pipeline_state.py b/SYS/pipeline_state.py new file mode 100644 index 0000000..a10a246 --- /dev/null +++ b/SYS/pipeline_state.py @@ -0,0 +1,1208 @@ +""" +Pipeline state management for cmdlet. +""" +from __future__ import annotations + +import sys +import time +import re +from contextlib import contextmanager +from dataclasses import dataclass, field +from contextvars import ContextVar +from typing import Any, Dict, List, Optional, Sequence, Callable +from SYS.models import PipelineStageContext +from SYS.logger import log, debug, debug_panel, is_debug_enabled +import logging + +logger = logging.getLogger(__name__) + +# SYS.worker deferred: ffmpeg+attr+rich (~260ms) loaded lazily on first pipeline run. +_worker_mod: Any = None +# SYS.cli_parsing deferred: prompt_toolkit (~300ms) loaded lazily on first selection. +_cli_parsing_mod: Any = None +# SYS.result_table deferred: textual (~140ms) loaded lazily on first Table use. +_result_table_mod: Any = None + + +def _worker() -> Any: + global _worker_mod + if _worker_mod is None: + import SYS.worker as _m + + _worker_mod = _m + return _worker_mod + + +def _cli_parsing() -> Any: + global _cli_parsing_mod + if _cli_parsing_mod is None: + import SYS.cli_parsing as _m + + _cli_parsing_mod = _m + return _cli_parsing_mod + + +def _result_table() -> Any: + global _result_table_mod + if _result_table_mod is None: + from SYS.result_table import Table as _T + + _result_table_mod = _T + return _result_table_mod + + +HELP_EXAMPLE_SOURCE_COMMANDS = { + ".help-example", + "help-example", +} + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- + + +def _emit_selection_debug_panel( + *, + selection_token: Any, + selection_indices: Sequence[int], + item_count: int, + filtered_count: int, + stage_table_present: bool, + display_table_present: bool, + stage_is_last: bool, + row_action: Optional[Sequence[Any]] = None, + downstream_stages: Optional[Sequence[Sequence[Any]]] = None, + mode: Optional[str] = None, +) -> None: + if not is_debug_enabled(): + return + + try: + rows: List[tuple[str, Any]] = [ + ("selection", str(selection_token or "")), + ("indices", [int(idx) + 1 for idx in (selection_indices or [])]), + ("items", int(item_count)), + ("filtered", int(filtered_count)), + ("stage_table", bool(stage_table_present)), + ("display_table", bool(display_table_present)), + ("stage_is_last", bool(stage_is_last)), + ("downstream_stages", len(list(downstream_stages or []))), + ] + if mode: + rows.insert(1, ("mode", str(mode))) + if row_action: + rows.append( + ("row_action", " ".join(str(part) for part in row_action if part is not None)) + ) + + debug_panel( + f"Selection replay {selection_token}", + rows, + border_style="magenta", + ) + except Exception: + pass + + +def set_live_progress(progress_ui: Any) -> None: + state = _get_pipeline_state() + state.live_progress = progress_ui + + +def get_live_progress() -> Any: + state = _get_pipeline_state() + return state.live_progress + + +def set_progress_event_callback(callback: Any) -> None: + state = _get_pipeline_state() + state.progress_event_callback = callback + + +def get_progress_event_callback() -> Any: + state = _get_pipeline_state() + return state.progress_event_callback + + +@contextmanager +def suspend_live_progress() -> Any: + ui = get_live_progress() + paused = False + try: + if ui is not None and hasattr(ui, "pause"): + try: + ui.pause() + paused = True + except Exception as exc: + logger.exception("Failed to pause live progress UI: %s", exc) + paused = False + yield + finally: + if get_pipeline_stop() is None: + if paused and ui is not None and hasattr(ui, "resume"): + try: + ui.resume() + except Exception: + logger.exception("Failed to resume live progress UI after suspend") + + +def _is_selectable_table(table: Any) -> bool: + return table is not None and not getattr(table, "no_choice", False) + + +# --------------------------------------------------------------------------- +# PipelineState +# --------------------------------------------------------------------------- + + +@dataclass +class PipelineState: + current_context: Optional[PipelineStageContext] = None + last_search_query: Optional[str] = None + pipeline_refreshed: bool = False + last_items: List[Any] = field(default_factory=list) + last_result_table: Optional[Any] = None + last_result_items: List[Any] = field(default_factory=list) + last_result_subject: Optional[Any] = None + result_table_history: List[tuple[Optional[Any], List[Any], Optional[Any]]] = field( + default_factory=list + ) + result_table_forward: List[tuple[Optional[Any], List[Any], Optional[Any]]] = field( + default_factory=list + ) + current_stage_table: Optional[Any] = None + display_items: List[Any] = field(default_factory=list) + display_table: Optional[Any] = None + display_subject: Optional[Any] = None + last_selection: List[int] = field(default_factory=list) + pipeline_command_text: str = "" + current_cmdlet_name: str = "" + current_stage_text: str = "" + pipeline_values: Dict[str, Any] = field(default_factory=dict) + pending_pipeline_tail: List[List[str]] = field(default_factory=list) + pending_pipeline_source: Optional[str] = None + ui_library_refresh_callback: Optional[Any] = None + pipeline_stop: Optional[Dict[str, Any]] = None + live_progress: Any = None + last_execution_result: Dict[str, Any] = field(default_factory=dict) + progress_event_callback: Any = None + + def reset(self) -> None: + self.current_context = None + self.last_search_query = None + self.pipeline_refreshed = False + self.last_items = [] + self.last_result_table = None + self.last_result_items = [] + self.last_result_subject = None + self.result_table_history = [] + self.result_table_forward = [] + self.current_stage_table = None + self.display_items = [] + self.display_table = None + self.display_subject = None + self.last_selection = [] + self.pipeline_command_text = "" + self.current_cmdlet_name = "" + self.current_stage_text = "" + self.pipeline_values = {} + self.pending_pipeline_tail = [] + self.pending_pipeline_source = None + self.ui_library_refresh_callback = None + self.pipeline_stop = None + self.live_progress = None + self.last_execution_result = {} + self.progress_event_callback = None + + +# --------------------------------------------------------------------------- +# ContextVar and global fallback +# --------------------------------------------------------------------------- + +_CTX_STATE: ContextVar[Optional[PipelineState]] = ContextVar("_pipeline_state", default=None) +_GLOBAL_STATE: PipelineState = PipelineState() + + +def _get_pipeline_state() -> PipelineState: + state = _CTX_STATE.get() + return state if state is not None else _GLOBAL_STATE + + +@contextmanager +def new_pipeline_state() -> Any: + token = _CTX_STATE.set(PipelineState()) + try: + yield _CTX_STATE.get() + finally: + _CTX_STATE.reset(token) + + +def get_pipeline_state() -> PipelineState: + return _get_pipeline_state() + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MAX_RESULT_TABLE_HISTORY = 20 +PIPELINE_MISSING = object() + + +# --------------------------------------------------------------------------- +# Pipeline stop +# --------------------------------------------------------------------------- + + +def request_pipeline_stop(*, reason: str = "", exit_code: int = 0) -> None: + state = _get_pipeline_state() + state.pipeline_stop = { + "reason": str(reason or "").strip(), + "exit_code": int(exit_code), + } + + +def get_pipeline_stop() -> Optional[Dict[str, Any]]: + state = _get_pipeline_state() + return state.pipeline_stop + + +def clear_pipeline_stop() -> None: + state = _get_pipeline_state() + state.pipeline_stop = None + + +# --------------------------------------------------------------------------- +# Stage context +# --------------------------------------------------------------------------- + + +def set_stage_context(context: Optional[PipelineStageContext]) -> None: + state = _get_pipeline_state() + state.current_context = context + + +def get_stage_context() -> Optional[PipelineStageContext]: + state = _get_pipeline_state() + return state.current_context + + +# --------------------------------------------------------------------------- +# Emit helpers +# --------------------------------------------------------------------------- + + +def emit(obj: Any) -> None: + ctx = _get_pipeline_state().current_context + if ctx is not None: + ctx.emit(obj) + + +def emit_list(objects: List[Any]) -> None: + ctx = _get_pipeline_state().current_context + if ctx is not None: + ctx.emit(objects) + + +def print_if_visible(*args: Any, file: Any = None, **kwargs: Any) -> None: + try: + ctx = _get_pipeline_state().current_context + should_print = (ctx is None) or (ctx and ctx.is_last_stage) + if file is not None: + should_print = True + if should_print: + log(*args, **kwargs) if file is None else log(*args, file=file, **kwargs) + except Exception: + logger.exception("Error in print_if_visible") + + +# --------------------------------------------------------------------------- +# Pipeline values +# --------------------------------------------------------------------------- + + +def store_value(key: str, value: Any) -> None: + if not isinstance(key, str): + return + text = key.strip().lower() + if not text: + return + try: + state = _get_pipeline_state() + state.pipeline_values[text] = value + except Exception: + logger.exception("Failed to store pipeline value '%s'", key) + + +def load_value(key: str, default: Any = None) -> Any: + if not isinstance(key, str): + return default + text = key.strip() + if not text: + return default + parts = [segment.strip() for segment in text.split(".") if segment.strip()] + if not parts: + return default + root_key = parts[0].lower() + state = _get_pipeline_state() + container = state.pipeline_values.get(root_key, PIPELINE_MISSING) + if container is PIPELINE_MISSING: + return default + if len(parts) == 1: + return container + + current: Any = container + for fragment in parts[1:]: + if isinstance(current, dict): + fragment_lower = fragment.lower() + if fragment in current: + current = current[fragment] + continue + match = PIPELINE_MISSING + for key_name, value in current.items(): + if isinstance(key_name, str) and key_name.lower() == fragment_lower: + match = value + break + if match is PIPELINE_MISSING: + return default + current = match + continue + if isinstance(current, (list, tuple)): + if fragment.isdigit(): + try: + idx = int(fragment) + except ValueError: + return default + if 0 <= idx < len(current): + current = current[idx] + continue + return default + if hasattr(current, fragment): + try: + current = getattr(current, fragment) + continue + except Exception: + return default + return default + return current + + +# --------------------------------------------------------------------------- +# Execution result +# --------------------------------------------------------------------------- + + +def set_last_execution_result( + *, + status: str, + error: str = "", + command_text: str = "", +) -> None: + state = _get_pipeline_state() + text_status = str(status or "").strip().lower() or "unknown" + state.last_execution_result = { + "status": text_status, + "success": text_status == "completed", + "error": str(error or "").strip(), + "command_text": str(command_text or "").strip(), + "finished_at": time.time(), + } + + +def get_last_execution_result() -> Dict[str, Any]: + state = _get_pipeline_state() + payload = state.last_execution_result + return dict(payload) if isinstance(payload, dict) else {} + + +# --------------------------------------------------------------------------- +# Pending pipeline tail +# --------------------------------------------------------------------------- + + +def set_pending_pipeline_tail( + stages: Optional[Sequence[Sequence[str]]], + source_command: Optional[str] = None, +) -> None: + state = _get_pipeline_state() + try: + pending: List[List[str]] = [] + for stage in stages or []: + if isinstance(stage, (list, tuple)): + pending.append([str(token) for token in stage]) + state.pending_pipeline_tail = pending + clean_source = (source_command or "").strip() + state.pending_pipeline_source = clean_source if clean_source else None + except Exception: + logger.exception("Failed to set pending pipeline tail; keeping existing pending tail") + + +def get_pending_pipeline_tail() -> List[List[str]]: + state = _get_pipeline_state() + return [list(stage) for stage in state.pending_pipeline_tail] + + +def get_pending_pipeline_source() -> Optional[str]: + state = _get_pipeline_state() + return state.pending_pipeline_source + + +def clear_pending_pipeline_tail() -> None: + state = _get_pipeline_state() + state.pending_pipeline_tail = [] + state.pending_pipeline_source = None + + +# --------------------------------------------------------------------------- +# Reset +# --------------------------------------------------------------------------- + + +def reset() -> None: + state = _get_pipeline_state() + state.reset() + + +# --------------------------------------------------------------------------- +# Emitted items +# --------------------------------------------------------------------------- + + +def get_emitted_items() -> List[Any]: + state = _get_pipeline_state() + ctx = state.current_context + if ctx is not None: + return list(ctx.emits) + return [] + + +def clear_emits() -> None: + state = _get_pipeline_state() + ctx = state.current_context + if ctx is not None: + ctx.emits.clear() + + +# --------------------------------------------------------------------------- +# Selection +# --------------------------------------------------------------------------- + + +def set_last_selection(indices: Sequence[int]) -> None: + state = _get_pipeline_state() + state.last_selection = list(indices or []) + + +def get_last_selection() -> List[int]: + state = _get_pipeline_state() + return list(state.last_selection) + + +def clear_last_selection() -> None: + state = _get_pipeline_state() + state.last_selection = [] + + +# --------------------------------------------------------------------------- +# Command text +# --------------------------------------------------------------------------- + + +def set_current_command_text(command_text: Optional[str]) -> None: + state = _get_pipeline_state() + state.pipeline_command_text = (command_text or "").strip() + + +def get_current_command_text(default: str = "") -> str: + state = _get_pipeline_state() + text = state.pipeline_command_text.strip() + return text if text else default + + +def clear_current_command_text() -> None: + state = _get_pipeline_state() + state.pipeline_command_text = "" + + +# --------------------------------------------------------------------------- +# Pipeline text splitting +# --------------------------------------------------------------------------- + + +def split_pipeline_text(pipeline_text: str) -> List[str]: + text = str(pipeline_text or "") + if not text: + return [] + + stages: List[str] = [] + buf: List[str] = [] + quote: Optional[str] = None + escape = False + + for ch in text: + if escape: + buf.append(ch) + escape = False + continue + + if ch == "\\" and quote is not None: + buf.append(ch) + escape = True + continue + + if ch in ('"', "'"): + if quote is None: + quote = ch + elif quote == ch: + quote = None + buf.append(ch) + continue + + if ch == "|" and quote is None: + stages.append("".join(buf).strip()) + buf = [] + continue + + buf.append(ch) + + tail = "".join(buf).strip() + if tail: + stages.append(tail) + return [s for s in stages if s] + + +def get_current_command_stages() -> List[str]: + return split_pipeline_text(get_current_command_text("")) + + +# --------------------------------------------------------------------------- +# Stage text +# --------------------------------------------------------------------------- + + +def set_current_stage_text(stage_text: Optional[str]) -> None: + state = _get_pipeline_state() + state.current_stage_text = str(stage_text or "").strip() + + +def get_current_stage_text(default: str = "") -> str: + state = _get_pipeline_state() + text = state.current_stage_text.strip() + return text if text else default + + +def clear_current_stage_text() -> None: + state = _get_pipeline_state() + state.current_stage_text = "" + + +# --------------------------------------------------------------------------- +# Cmdlet name +# --------------------------------------------------------------------------- + + +def set_current_cmdlet_name(cmdlet_name: Optional[str]) -> None: + state = _get_pipeline_state() + state.current_cmdlet_name = str(cmdlet_name or "").strip() + + +def get_current_cmdlet_name(default: str = "") -> str: + state = _get_pipeline_state() + text = state.current_cmdlet_name.strip() + return text if text else default + + +def clear_current_cmdlet_name() -> None: + state = _get_pipeline_state() + state.current_cmdlet_name = "" + + +# --------------------------------------------------------------------------- +# Search query +# --------------------------------------------------------------------------- + + +def set_search_query(query: Optional[str]) -> None: + state = _get_pipeline_state() + state.last_search_query = query + + +def get_search_query() -> Optional[str]: + state = _get_pipeline_state() + return state.last_search_query + + +# --------------------------------------------------------------------------- +# Refresh tracking +# --------------------------------------------------------------------------- + + +def set_pipeline_refreshed(refreshed: bool) -> None: + state = _get_pipeline_state() + state.pipeline_refreshed = refreshed + + +def was_pipeline_refreshed() -> bool: + state = _get_pipeline_state() + return state.pipeline_refreshed + + +# --------------------------------------------------------------------------- +# Last items +# --------------------------------------------------------------------------- + + +def set_last_items(items: list) -> None: + state = _get_pipeline_state() + state.last_items = list(items) if items else [] + + +def get_last_items() -> List[Any]: + state = _get_pipeline_state() + return list(state.last_items) + + +# --------------------------------------------------------------------------- +# UI library refresh callback +# --------------------------------------------------------------------------- + + +def set_ui_library_refresh_callback(callback: Any) -> None: + state = _get_pipeline_state() + state.ui_library_refresh_callback = callback + + +def get_ui_library_refresh_callback() -> Optional[Any]: + state = _get_pipeline_state() + return state.ui_library_refresh_callback + + +def trigger_ui_library_refresh(library_filter: str = "local") -> None: + callback = get_ui_library_refresh_callback() + if callback: + try: + callback(library_filter) + except Exception as e: + print( + f"[trigger_ui_library_refresh] Error calling refresh callback: {e}", + file=sys.stderr, + ) + + +# --------------------------------------------------------------------------- +# Result table: set (push to history) +# --------------------------------------------------------------------------- + + +def set_last_result_table( + result_table: Optional[Any], + items: Optional[List[Any]] = None, + subject: Optional[Any] = None, +) -> None: + state = _get_pipeline_state() + + if state.last_result_table is not None: + state.result_table_history.append( + ( + state.last_result_table, + state.last_result_items, + state.last_result_subject, + ) + ) + if len(state.result_table_history) > MAX_RESULT_TABLE_HISTORY: + state.result_table_history.pop(0) + + state.display_items = [] + state.display_table = None + state.display_subject = None + state.last_result_table = result_table + state.last_result_items = items or [] + state.last_result_subject = subject + + if ( + result_table is not None + and hasattr(result_table, "sort_by_title") + and not getattr(result_table, "preserve_order", False) + ): + try: + result_table.sort_by_title() + if state.last_result_items and hasattr(result_table, "rows"): + sorted_items: List[Any] = [] + for row in result_table.rows: + src_idx = getattr(row, "source_index", None) + if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items): + sorted_items.append(state.last_result_items[src_idx]) + if result_table.rows and len(sorted_items) == len(result_table.rows): + state.last_result_items = sorted_items + except Exception: + logger.exception("Failed to sort result_table and reorder items") + + +# --------------------------------------------------------------------------- +# Result table: overlay (no history push) +# --------------------------------------------------------------------------- + + +def set_last_result_table_overlay( + result_table: Optional[Any], + items: Optional[List[Any]] = None, + subject: Optional[Any] = None, +) -> None: + state = _get_pipeline_state() + state.display_table = result_table + state.display_items = items or [] + state.display_subject = subject + + +# --------------------------------------------------------------------------- +# Result table: items only (no history, no table) +# --------------------------------------------------------------------------- + + +def set_last_result_items_only(items: Optional[List[Any]]) -> None: + state = _get_pipeline_state() + state.display_items = items or [] + state.display_table = None + state.display_subject = None + + +# --------------------------------------------------------------------------- +# Result table history navigation (consolidated) +# --------------------------------------------------------------------------- + + +def _restore_result_table(forward: bool) -> bool: + """Consolidated implementation for restore_previous/restore_next_result_table.""" + state = _get_pipeline_state() + pop_stack = state.result_table_forward if forward else state.result_table_history + push_stack = state.result_table_history if forward else state.result_table_forward + label = "restore_next_result_table" if forward else "restore_previous_result_table" + + if state.display_items or state.display_table or state.display_subject is not None: + state.display_items = [] + state.display_table = None + state.display_subject = None + if state.last_result_table is not None: + state.current_stage_table = state.last_result_table + return True + if not pop_stack: + state.current_stage_table = state.last_result_table + return True + + if not pop_stack: + return False + + push_stack.append( + (state.last_result_table, state.last_result_items, state.last_result_subject) + ) + + prev = pop_stack.pop() + if isinstance(prev, tuple) and len(prev) >= 3: + state.last_result_table, state.last_result_items, state.last_result_subject = ( + prev[0], + prev[1], + prev[2], + ) + elif isinstance(prev, tuple) and len(prev) == 2: + state.last_result_table, state.last_result_items = prev + state.last_result_subject = None + else: + state.last_result_table, state.last_result_items, state.last_result_subject = None, [], None + + state.display_items = [] + state.display_table = None + state.display_subject = None + state.current_stage_table = state.last_result_table + + try: + debug_table_state(label) + except Exception: + logger.exception("Failed to debug_table_state during %s", label) + + return True + + +def restore_previous_result_table() -> bool: + return _restore_result_table(forward=False) + + +def restore_next_result_table() -> bool: + return _restore_result_table(forward=True) + + +# --------------------------------------------------------------------------- +# Display table accessors +# --------------------------------------------------------------------------- + + +def get_display_table() -> Optional[Any]: + state = _get_pipeline_state() + return state.display_table + + +def get_last_result_subject() -> Optional[Any]: + state = _get_pipeline_state() + if state.display_subject is not None: + return state.display_subject + return state.last_result_subject + + +def get_last_result_table() -> Optional[Any]: + state = _get_pipeline_state() + return state.last_result_table + + +def get_last_result_items() -> List[Any]: + state = _get_pipeline_state() + if state.display_items: + if state.display_table is not None and not _is_selectable_table(state.display_table): + return [] + return state.display_items + if state.last_result_table is None: + return state.last_result_items + if _is_selectable_table(state.last_result_table): + return state.last_result_items + return [] + + +# --------------------------------------------------------------------------- +# Debug table state +# --------------------------------------------------------------------------- + + +def debug_table_state(label: str = "") -> None: + if not is_debug_enabled(): + return + + state = _get_pipeline_state() + + def _tbl(name: str, t: Any) -> None: + if t is None: + debug(f"[table] {name}: None") + return + try: + table_type = getattr(t, "table", None) + except Exception: + table_type = None + try: + title = getattr(t, "title", None) + except Exception: + title = None + try: + src_cmd = getattr(t, "source_command", None) + except Exception: + src_cmd = None + try: + src_args = getattr(t, "source_args", None) + except Exception: + src_args = None + try: + no_choice = bool(getattr(t, "no_choice", False)) + except Exception: + no_choice = False + try: + preserve_order = bool(getattr(t, "preserve_order", False)) + except Exception: + preserve_order = False + try: + row_count = len(getattr(t, "rows", []) or []) + except Exception: + row_count = 0 + try: + meta = ( + t.get_table_metadata() + if hasattr(t, "get_table_metadata") + else getattr(t, "table_metadata", None) + ) + except Exception: + meta = None + meta_keys = list(meta.keys()) if isinstance(meta, dict) else [] + + debug( + f"[table] {name}: id={id(t)} class={type(t).__name__} title={repr(title)} table={repr(table_type)} rows={row_count} " + f"source={repr(src_cmd)} source_args={repr(src_args)} no_choice={no_choice} preserve_order={preserve_order} meta_keys={meta_keys}" + ) + + if label: + debug(f"[table] state: {label}") + _tbl("display_table", getattr(state, "display_table", None)) + _tbl("current_stage_table", getattr(state, "current_stage_table", None)) + _tbl("last_result_table", getattr(state, "last_result_table", None)) + + try: + debug( + f"[table] buffers: display_items={len(state.display_items or [])} last_result_items={len(state.last_result_items or [])} " + f"history={len(state.result_table_history or [])} forward={len(state.result_table_forward or [])} last_selection={list(state.last_selection or [])}" + ) + except Exception: + logger.exception("Failed to debug_table_state buffers summary") + + +# --------------------------------------------------------------------------- +# Selectable result items +# --------------------------------------------------------------------------- + + +def get_last_selectable_result_items() -> List[Any]: + state = _get_pipeline_state() + if state.last_result_table is None: + return list(state.last_result_items) + if _is_selectable_table(state.last_result_table): + return list(state.last_result_items) + return [] + + +# --------------------------------------------------------------------------- +# Table source command/args accessors (consolidated helpers) +# --------------------------------------------------------------------------- + + +def _get_table_source_command(table: Any) -> Optional[str]: + if table is not None and _is_selectable_table(table) and hasattr(table, "source_command"): + return getattr(table, "source_command") + return None + + +def _get_table_source_args(table: Any) -> List[str]: + if table is not None and _is_selectable_table(table) and hasattr(table, "source_args"): + return getattr(table, "source_args") or [] + return [] + + +def _get_table_row_selection_args(table: Any, row_index: int) -> Optional[List[str]]: + if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): + rows = table.rows + if 0 <= row_index < len(rows): + row = rows[row_index] + if hasattr(row, "selection_args"): + return getattr(row, "selection_args") + return None + + +def _get_table_row_selection_action(table: Any, row_index: int) -> Optional[List[str]]: + if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): + rows = table.rows + if 0 <= row_index < len(rows): + row = rows[row_index] + if hasattr(row, "selection_action"): + return getattr(row, "selection_action") + return None + + +# --------------------------------------------------------------------------- +# Public table accessor functions (delegate to helpers) +# --------------------------------------------------------------------------- + + +def get_last_result_table_source_command() -> Optional[str]: + state = _get_pipeline_state() + return _get_table_source_command(state.last_result_table) + + +def get_last_result_table_source_args() -> List[str]: + state = _get_pipeline_state() + return _get_table_source_args(state.last_result_table) + + +def get_last_result_table_row_selection_args(row_index: int) -> Optional[List[str]]: + state = _get_pipeline_state() + return _get_table_row_selection_args(state.last_result_table, row_index) + + +def get_last_result_table_row_selection_action(row_index: int) -> Optional[List[str]]: + state = _get_pipeline_state() + return _get_table_row_selection_action(state.last_result_table, row_index) + + +def get_current_stage_table() -> Optional[Any]: + state = _get_pipeline_state() + return state.current_stage_table + + +def set_current_stage_table(result_table: Optional[Any]) -> None: + state = _get_pipeline_state() + state.current_stage_table = result_table + + +def get_current_stage_table_source_command() -> Optional[str]: + state = _get_pipeline_state() + return _get_table_source_command(state.current_stage_table) + + +def get_current_stage_table_source_args() -> List[str]: + state = _get_pipeline_state() + return _get_table_source_args(state.current_stage_table) + + +def get_current_stage_table_row_selection_args(row_index: int) -> Optional[List[str]]: + state = _get_pipeline_state() + return _get_table_row_selection_args(state.current_stage_table, row_index) + + +def get_current_stage_table_row_selection_action(row_index: int) -> Optional[List[str]]: + state = _get_pipeline_state() + return _get_table_row_selection_action(state.current_stage_table, row_index) + + +def get_current_stage_table_row_source_index(row_index: int) -> Optional[int]: + state = _get_pipeline_state() + table = state.current_stage_table + if table is not None and _is_selectable_table(table) and hasattr(table, "rows"): + rows = table.rows + if 0 <= row_index < len(rows): + row = rows[row_index] + return getattr(row, "source_index", None) + return None + + +def clear_last_result() -> None: + state = _get_pipeline_state() + state.last_result_table = None + state.last_result_items = [] + state.last_result_subject = None + + +# --------------------------------------------------------------------------- +# Pipeline token splitting (used by executor) +# --------------------------------------------------------------------------- + + +def _split_pipeline_tokens(tokens: Sequence[str]) -> List[List[str]]: + stages: List[List[str]] = [] + current: List[str] = [] + for token in tokens: + if token == "|": + if current: + stages.append(current) + current = [] + continue + current.append(str(token)) + if current: + stages.append(current) + return [stage for stage in stages if stage] + + +# --------------------------------------------------------------------------- +# Module __all__ +# --------------------------------------------------------------------------- + +__all__ = [ + # State class + "PipelineState", + # ContextVar and factories + "_CTX_STATE", + "_GLOBAL_STATE", + "_get_pipeline_state", + "get_pipeline_state", + "new_pipeline_state", + # Constants + "MAX_RESULT_TABLE_HISTORY", + "PIPELINE_MISSING", + "HELP_EXAMPLE_SOURCE_COMMANDS", + # Deferred module loaders + "_worker", + "_cli_parsing", + "_result_table", + # Live progress + "set_live_progress", + "get_live_progress", + "set_progress_event_callback", + "get_progress_event_callback", + "suspend_live_progress", + # Pipeline stop + "request_pipeline_stop", + "get_pipeline_stop", + "clear_pipeline_stop", + # Stage context + "set_stage_context", + "get_stage_context", + # Emit helpers + "emit", + "emit_list", + "print_if_visible", + # Pipeline values + "store_value", + "load_value", + # Execution result + "set_last_execution_result", + "get_last_execution_result", + # Pending pipeline tail + "set_pending_pipeline_tail", + "get_pending_pipeline_tail", + "get_pending_pipeline_source", + "clear_pending_pipeline_tail", + # Reset + "reset", + # Emitted items + "get_emitted_items", + "clear_emits", + # Selection + "set_last_selection", + "get_last_selection", + "clear_last_selection", + # Command text + "set_current_command_text", + "get_current_command_text", + "clear_current_command_text", + # Pipeline splitting + "split_pipeline_text", + "get_current_command_stages", + "_split_pipeline_tokens", + # Stage text + "set_current_stage_text", + "get_current_stage_text", + "clear_current_stage_text", + # Cmdlet name + "set_current_cmdlet_name", + "get_current_cmdlet_name", + "clear_current_cmdlet_name", + # Search query + "set_search_query", + "get_search_query", + # Refresh + "set_pipeline_refreshed", + "was_pipeline_refreshed", + # Last items + "set_last_items", + "get_last_items", + # UI library + "set_ui_library_refresh_callback", + "get_ui_library_refresh_callback", + "trigger_ui_library_refresh", + # Result table setters + "set_last_result_table", + "set_last_result_table_overlay", + "set_last_result_items_only", + # History navigation + "restore_previous_result_table", + "restore_next_result_table", + # Display getters + "get_display_table", + "get_last_result_subject", + "get_last_result_table", + "get_last_result_items", + "get_last_selectable_result_items", + # Table accessors + "get_last_result_table_source_command", + "get_last_result_table_source_args", + "get_last_result_table_row_selection_args", + "get_last_result_table_row_selection_action", + "get_current_stage_table", + "set_current_stage_table", + "get_current_stage_table_source_command", + "get_current_stage_table_source_args", + "get_current_stage_table_row_selection_args", + "get_current_stage_table_row_selection_action", + "get_current_stage_table_row_source_index", + # Clear + "clear_last_result", + # Debug + "debug_table_state", + # Helpers (used internally by executor) + "_emit_selection_debug_panel", + "_is_selectable_table", + "_get_table_source_command", + "_get_table_source_args", + "_get_table_row_selection_args", + "_get_table_row_selection_action", +] diff --git a/SYS/result_table.py b/SYS/result_table.py index b758616..c5243f9 100644 --- a/SYS/result_table.py +++ b/SYS/result_table.py @@ -45,22 +45,6 @@ def _rich(): return _rich_mod -# Optional Textual imports - lazily loaded to avoid pulling in ~300ms of textual -# at import time when the TUI is not being used. -import importlib.util as _importlib_util -TEXTUAL_AVAILABLE: bool = _importlib_util.find_spec("textual") is not None - -# Tree is populated lazily on first call to build_metadata_tree(). -_textual_Tree: Any = None - - -def _get_textual_Tree() -> Any: - global _textual_Tree - if _textual_Tree is None: - from textual.widgets import Tree as _Tree - _textual_Tree = _Tree - return _textual_Tree - # Import ResultModel from the API for typing; avoid runtime redefinition issues from typing import TYPE_CHECKING @@ -69,6 +53,8 @@ if TYPE_CHECKING: else: ResultModel = None # type: ignore[assignment] + + # Reuse the existing format_bytes helper under a clearer alias from SYS.utils import format_bytes as format_mb @@ -587,30 +573,6 @@ class InputOption: } -@dataclass -class TUIResultCard: - """Represents a result as a UI card with title, metadata, and actions. - - Used in hub-ui and TUI contexts to render individual search results - as grouped components with visual structure. - """ - - title: str - subtitle: Optional[str] = None - metadata: Optional[Dict[str, str]] = None - media_kind: Optional[str] = None - tag: Optional[List[str]] = None - file_hash: Optional[str] = None - file_size: Optional[str] = None - duration: Optional[str] = None - - def __post_init__(self): - """Initialize default values.""" - if self.metadata is None: - self.metadata = {} - if self.tag is None: - self.tag = [] - @dataclass class Column: @@ -2113,122 +2075,6 @@ class Table: return self.rows[idx] return None - # TUI-specific formatting methods - - def to_datatable_rows(self, source: str = "unknown") -> List[List[str]]: - """Convert results to rows suitable for Textual DataTable widget. - - Args: - source: Source type for formatting context (openlibrary, soulseek, etc.) - - Returns: - List of row value lists - """ - rows = [] - for result in self.rows: - row_values = self._format_datatable_row(result, source) - rows.append(row_values) - return rows - - def _format_datatable_row(self, - row: Row, - source: str = "unknown") -> List[str]: - """Format a ResultRow for DataTable display. - - Args: - row: ResultRow to format - source: Source type - - Returns: - List of column values as strings - """ - # Extract values from row columns - values = [col.value for col in row.columns] - - # Truncate to reasonable lengths for table display - return [v[:60] if len(v) > 60 else v for v in values] - - def to_result_cards(self) -> List[TUIResultCard]: - """Convert all rows to TUIResultCard objects for card-based UI display. - - Returns: - List of TUIResultCard objects - """ - cards = [] - for row in self.rows: - card = self._row_to_card(row) - cards.append(card) - return cards - - def _row_to_card(self, row: Row) -> TUIResultCard: - """Convert a ResultRow to a TUIResultCard. - - Args: - row: ResultRow to convert - - Returns: - TUIResultCard with extracted metadata - """ - # Build metadata dict from row columns - metadata = {} - title = "" - - for col in row.columns: - if col.name.lower() == "title": - title = col.value - metadata[col.name] = col.value - - # Extract tag values if present - tag = [] - if "Tag" in metadata: - tag_val = metadata["Tag"] - if tag_val: - tag = [t.strip() for t in tag_val.split(",")][:5] - - # Try to find useful metadata fields - subtitle = metadata.get("Artist", metadata.get("Author", "")) - media_kind = metadata.get("Type", metadata.get("Media Kind", "")) - file_size = metadata.get("Size", "") - duration = metadata.get("Duration", "") - file_hash = metadata.get("Hash", "") - - return TUIResultCard( - title=title or "Unknown", - subtitle=subtitle, - metadata=metadata, - media_kind=media_kind, - tag=tag, - file_hash=file_hash or None, - file_size=file_size or None, - duration=duration or None, - ) - - def build_metadata_tree(self, tree_widget: "Tree") -> None: - """Populate a Textual Tree widget with result metadata hierarchy. - - Args: - tree_widget: Textual Tree widget to populate - - Raises: - ImportError: If Textual not available - """ - if not TEXTUAL_AVAILABLE: - raise ImportError("Textual not available for tree building") - - tree_widget.reset(self.title or "Results") - root = tree_widget.root - - # Add each row as a top-level node - for i, row in enumerate(self.rows, 1): - row_node = root.add(f"[bold]Result {i}[/bold]") - - # Add columns as children - for col in row.columns: - value_str = col.value - if len(value_str) > 100: - value_str = value_str[:97] + "..." - row_node.add_leaf(f"[cyan]{col.name}[/cyan]: {value_str}") - def _format_size(size: Any, integer_only: bool = False) -> str: """Format file size as human-readable string. @@ -2577,9 +2423,46 @@ class ItemDetailView(Table): return f"{label[:max_len - 3]}..." return label + def _looks_like_local_path(value: Any) -> bool: + text = str(value or "").strip() + if not text: + return False + try: + p = Path(text) + if p.exists(): + return True + return text.startswith("/") or text.startswith("\\\\") or (len(text) >= 2 and text[1] == ":" and text[2:3] in ("\\", "/")) + except Exception: + return False + + def _short_path_label(path_value: str, *, max_len: int = 68) -> str: + text = str(path_value or "").strip() + if not text: + return "" + if len(text) <= max_len: + return text + parts = text.replace("\\", "/").rstrip("/").split("/") + if len(parts) >= 3: + head = parts[0] + tail = parts[-1] + mid = "..." + label = f"{head}/{mid}/{tail}" + if len(label) > max_len: + return f"{text[:max_len - 3]}..." + return label + return f"{text[:max_len - 3]}..." + def _render_detail_value(key: str, value: Any) -> Any: - # Keep terminal links compact while preserving full click target. - if str(key or "").strip().lower() in {"path", "url"} and _looks_like_http_url(value): + key_lower = str(key or "").strip().lower() + if key_lower == "path" and _looks_like_local_path(value): + full_path = str(value).strip() + try: + file_uri = Path(full_path).as_uri() + except Exception: + return str(value) + label = _short_path_label(full_path) + return _rich().Text(label, style=f"underline cyan link {file_uri}") + if key_lower in {"path", "url"} and _looks_like_http_url(value): full_url = str(value).strip() label = _short_link_label(full_url) return _rich().Text(label, style=f"underline cyan link {full_url}") diff --git a/SYS/rich_display.py b/SYS/rich_display.py index 3fe7122..00c064c 100644 --- a/SYS/rich_display.py +++ b/SYS/rich_display.py @@ -109,7 +109,7 @@ def show_plugin_config_panel( group = Group( Text("The following plugins are not configured and cannot be used:\n"), table, - Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.") + Text.from_markup("\nTo configure them, use the [bold cyan].config[/bold cyan] command in the REPL.") ) panel = Panel( diff --git a/SYS/worker_manager.py b/SYS/worker_manager.py index e4ee8e0..59c6dc4 100644 --- a/SYS/worker_manager.py +++ b/SYS/worker_manager.py @@ -260,6 +260,11 @@ class WorkerLoggingHandler(logging.StreamHandler): super().close() +_DEFAULT_STDOUT_FLUSH_BYTES = 4096 +_DEFAULT_STDOUT_FLUSH_INTERVAL = 0.75 +_REFRESH_THREAD_JOIN_TIMEOUT = 5 + + class WorkerManager: """Manages persistent worker tasks using the central medios.db.""" @@ -284,8 +289,8 @@ class WorkerManager: self._stdout_buffer_sizes: Dict[Tuple[str, str], int] = {} self._stdout_buffer_steps: Dict[Tuple[str, str], Optional[str]] = {} self._stdout_last_flush: Dict[Tuple[str, str], float] = {} - self._stdout_flush_bytes = 4096 - self._stdout_flush_interval = 0.75 + self._stdout_flush_bytes = _DEFAULT_STDOUT_FLUSH_BYTES + self._stdout_flush_interval = _DEFAULT_STDOUT_FLUSH_INTERVAL def __enter__(self): @@ -707,7 +712,7 @@ class WorkerManager: self._stop_refresh = True self._refresh_stop_event.set() if self.refresh_thread: - self.refresh_thread.join(timeout=5) + self.refresh_thread.join(timeout=_REFRESH_THREAD_JOIN_TIMEOUT) self.refresh_thread = None def _start_refresh_if_needed(self) -> None: diff --git a/TUI.py b/TUI.py deleted file mode 100644 index 1dcfd6a..0000000 --- a/TUI.py +++ /dev/null @@ -1,2466 +0,0 @@ -"""Modern Textual UI for driving Medeia-Macina pipelines.""" - -from __future__ import annotations - -import json -import os -import re -import sys -import subprocess -import time -from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple -from rich.text import Text - -from textual import on, work -from textual.app import App, ComposeResult -from textual.binding import Binding -from textual.events import Key -from textual.containers import Container, Horizontal, Vertical -from textual.screen import ModalScreen -from textual.widgets import ( - Button, - DataTable, - Footer, - Header, - Input, - Label, - OptionList, - Select, - Static, - TextArea, - Tree, -) -from textual.widgets.option_list import Option -try: - from textual.suggester import SuggestFromList -except Exception: # pragma: no cover - Textual version dependent - SuggestFromList = None # type: ignore[assignment] - -import logging -logger = logging.getLogger(__name__) - -BASE_DIR = Path(__file__).resolve().parent -REPO_ROOT = BASE_DIR -TUI_DIR = REPO_ROOT / "TUI" -for path in (REPO_ROOT, TUI_DIR): - str_path = str(path) - if str_path not in sys.path: - sys.path.insert(0, str_path) - -from SYS.pipeline_runner import PipelineRunResult # type: ignore # noqa: E402 -from SYS.result_table import Table, extract_hash_value, extract_store_value, get_result_table_row_style # type: ignore # noqa: E402 - -from SYS.config import load_config # type: ignore # noqa: E402 -from SYS.database import db -from PluginCore.backend_registry import BackendRegistry # type: ignore # noqa: E402 -from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402 -from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402 - -from SYS.pipeline_runner import PipelineRunner # type: ignore # noqa: E402 - - -def _dedup_preserve_order(items: List[str]) -> List[str]: - out: List[str] = [] - seen: set[str] = set() - for raw in items: - s = str(raw or "").strip() - if not s: - continue - key = s.lower() - if key in seen: - continue - seen.add(key) - out.append(s) - return out - - -def _extract_tag_names(emitted: Sequence[Any]) -> List[str]: - tags: List[str] = [] - for obj in emitted or []: - try: - if hasattr(obj, "tag_name"): - val = getattr(obj, "tag_name") - if val and isinstance(val, str): - tags.append(val) - continue - except Exception: - logger.exception("Error extracting tag_name in _extract_tag_names") - - if isinstance(obj, dict): - # Prefer explicit tag lists - tag_list = obj.get("tag") - if isinstance(tag_list, (list, tuple)): - for t in tag_list: - if isinstance(t, str) and t.strip(): - tags.append(t.strip()) - if tag_list: - continue - # Fall back to individual tag_name/value/name strings - for k in ("tag_name", "value", "name"): - v = obj.get(k) - if isinstance(v, str) and v.strip(): - tags.append(v.strip()) - break - continue - return _dedup_preserve_order(tags) - - -def _extract_tag_names_from_table(table: Any) -> List[str]: - if not table: - return [] - sources: List[Any] = [] - get_payloads = getattr(table, "get_payloads", None) - if callable(get_payloads): - try: - payloads = get_payloads() - if payloads: - sources.extend(payloads) - except Exception: - logger.exception("Error while calling table.get_payloads") - rows = getattr(table, "rows", []) or [] - for row in rows: - for col in getattr(row, "columns", []) or []: - if str(getattr(col, "name", "") or "").strip().lower() == "tag": - val = getattr(col, "value", None) - if val: - sources.append({"tag_name": val}) - if not sources: - return [] - return _extract_tag_names(sources) - - -class TextPopup(ModalScreen[None]): - - def __init__(self, *, title: str, text: str) -> None: - super().__init__() - self._title = str(title) - self._text = str(text or "") - - def compose(self) -> ComposeResult: - yield Static(self._title, id="popup-title") - yield TextArea(self._text, id="popup-text", read_only=True) - yield Button("Close", id="popup-close") - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "popup-close": - self.dismiss(None) - - -class ActionMenuPopup(ModalScreen[Optional[str]]): - - def __init__(self, *, actions: List[Tuple[str, str]]) -> None: - super().__init__() - self._actions = list(actions or []) - - def compose(self) -> ComposeResult: - yield Static("Actions", id="popup-title") - with Vertical(id="actions-list"): - if self._actions: - for index, (label, _key) in enumerate(self._actions): - yield Button(str(label), id=f"actions-btn-{index}") - else: - yield Static("No actions available", id="actions-empty") - with Horizontal(id="actions-footer"): - yield Button("Close", id="actions-close") - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "actions-close": - self.dismiss(None) - return - btn_id = str(getattr(event.button, "id", "") or "") - if not btn_id.startswith("actions-btn-"): - return - try: - index = int(btn_id.rsplit("-", 1)[-1]) - except Exception: - self.dismiss(None) - return - if 0 <= index < len(self._actions): - self.dismiss(str(self._actions[index][1])) - else: - self.dismiss(None) - - -class TagEditorPopup(ModalScreen[None]): - - def __init__( - self, - *, - seeds: Any, - store_name: str, - file_hash: Optional[str] - ) -> None: - super().__init__() - self._seeds = seeds - self._store = str(store_name or "").strip() - self._hash = str(file_hash or "").strip() if file_hash else "" - self._original_tags: List[str] = [] - self._status: Optional[Static] = None - self._editor: Optional[TextArea] = None - - def compose(self) -> ComposeResult: - yield Static("Tags", id="popup-title") - yield TextArea("", id="tags-editor") - with Horizontal(id="tags-buttons"): - yield Button("Save", id="tags-save") - yield Button("Close", id="tags-close") - yield Static("", id="tags-status") - - def on_mount(self) -> None: - self._status = self.query_one("#tags-status", Static) - self._editor = self.query_one("#tags-editor", TextArea) - self._set_status("Loading tags…") - self._load_tags_background() - - def _set_status(self, msg: str) -> None: - if self._status: - self._status.update(str(msg or "")) - - @work(thread=True) - def _load_tags_background(self) -> None: - app = self.app # PipelineHubApp - tags = self._fetch_tags_from_store() - if not tags: - try: - runner: PipelineRunner = getattr(app, "executor") - cmd = "@1 | metadata -get" - res = runner.run_pipeline(cmd, seeds=self._seeds, isolate=True) - tags = _extract_tag_names_from_table(getattr(res, "result_table", None)) - if not tags: - tags = _extract_tag_names(getattr(res, "emitted", [])) - except Exception as exc: - tags = [] - try: - app.call_from_thread( - self._set_status, - f"Error: {type(exc).__name__}: {exc}" - ) - except Exception: - self._set_status(f"Error: {type(exc).__name__}: {exc}") - self._original_tags = tags - try: - app.call_from_thread(self._apply_loaded_tags, tags) - except Exception: - self._apply_loaded_tags(tags) - - def _apply_loaded_tags(self, tags: List[str]) -> None: - if self._editor: - self._editor.text = "\n".join(tags) - self._set_status(f"Loaded {len(tags)} tag(s)") - - def _fetch_tags_from_store(self) -> Optional[List[str]]: - if not self._store or not self._hash: - return None - try: - cfg = load_config() or {} - except Exception: - cfg = {} - store_key = str(self._store or "").strip() - hash_value = str(self._hash or "").strip().lower() - if not store_key or not hash_value: - return None - try: - registry = BackendRegistry(config=cfg, suppress_debug=True) - except Exception: - return [] - match = None - normalized = store_key.lower() - for name in registry.list_backends(): - if str(name or "").strip().lower() == normalized: - match = name - break - if match is None: - return None - try: - backend = registry[match] - except KeyError: - return None - try: - tags, _src = backend.get_tag(hash_value, config=cfg) - if not tags: - return [] - filtered = [str(t).strip() for t in tags if str(t).strip()] - return _dedup_preserve_order(filtered) - except Exception: - return None - - def _parse_editor_tags(self) -> List[str]: - raw = "" - try: - raw = str(self._editor.text or "") if self._editor else "" - except Exception: - raw = "" - lines = [t.strip() for t in raw.replace("\r\n", "\n").split("\n")] - return _dedup_preserve_order([t for t in lines if t]) - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "tags-close": - self.dismiss(None) - return - if event.button.id == "tags-save": - self._save_tags() - - def _save_tags(self) -> None: - desired = self._parse_editor_tags() - current = _dedup_preserve_order(list(self._original_tags or [])) - - desired_set = {t.lower() - for t in desired} - current_set = {t.lower() - for t in current} - - to_add = [t for t in desired if t.lower() not in current_set] - to_del = [t for t in current if t.lower() not in desired_set] - - if not to_add and not to_del: - self._set_status("No changes") - return - - self._set_status("Saving…") - self._save_tags_background(to_add, to_del, desired) - - @work(thread=True) - def _save_tags_background( - self, - to_add: List[str], - to_del: List[str], - desired: List[str] - ) -> None: - app = self.app # PipelineHubApp - def _log_message(msg: str) -> None: - if not msg: - return - try: - app.call_from_thread(app._append_log_line, msg) - except Exception: - logger.exception("Failed to append log line from background thread") - - def _log_pipeline_command(stage: str, cmd: str) -> None: - if not cmd: - return - _log_message(f"tags-save: {stage}: {cmd}") - - def _log_pipeline_result(stage: str, result: PipelineRunResult | None) -> None: - if result is None: - return - status = "success" if getattr(result, "success", False) else "failed" - _log_message(f"tags-save: {stage} result: {status}") - error = str(getattr(result, "error", "") or "").strip() - if error: - _log_message(f"tags-save: {stage} error: {error}") - for attr in ("stdout", "stderr"): - raw = str(getattr(result, attr, "") or "").strip() - if not raw: - continue - for line in raw.splitlines(): - _log_message(f"tags-save: {stage} {attr}: {line}") - try: - runner: PipelineRunner = getattr(app, "executor") - store_tok = json.dumps(self._store) - query_chunk = f" -query {json.dumps(f'hash:{self._hash}')}" if self._hash else "" - - failures: List[str] = [] - - if to_del: - del_args = " ".join(json.dumps(t) for t in to_del) - del_cmd = f"metadata -delete -instance {store_tok}{query_chunk} {del_args}" - _log_pipeline_command("metadata-delete", del_cmd) - del_res = runner.run_pipeline(del_cmd, seeds=self._seeds, isolate=True) - _log_pipeline_result("metadata-delete", del_res) - if not getattr(del_res, "success", False): - failures.append( - str( - getattr(del_res, - "error", - "") or getattr(del_res, - "stderr", - "") or "metadata -delete failed" - ).strip() - ) - - if to_add: - add_args = " ".join(json.dumps(t) for t in to_add) - add_cmd = f"metadata -add -instance {store_tok}{query_chunk} {add_args}" - _log_pipeline_command("metadata-add", add_cmd) - add_res = runner.run_pipeline(add_cmd, seeds=self._seeds, isolate=True) - _log_pipeline_result("metadata-add", add_res) - if not getattr(add_res, "success", False): - failures.append( - str( - getattr(add_res, - "error", - "") or getattr(add_res, - "stderr", - "") or "metadata -add failed" - ).strip() - ) - - if failures: - msg = failures[0] - try: - app.call_from_thread(self._set_status, f"Error: {msg}") - except Exception: - self._set_status(f"Error: {msg}") - return - - reloaded = self._fetch_tags_from_store() - refreshed = reloaded is not None - tags_to_show = list(reloaded or []) if refreshed else list(desired) - self._original_tags = list(tags_to_show) - try: - app.call_from_thread(self._apply_loaded_tags, tags_to_show) - except Exception: - self._apply_loaded_tags(tags_to_show) - - def _refresh_overlay() -> None: - try: - app.refresh_tag_overlay( - self._store, - self._hash, - tags_to_show, - self._seeds, - ) - except Exception: - logger.exception("Failed to refresh tag overlay") - - try: - app.call_from_thread(_refresh_overlay) - except Exception: - _refresh_overlay() - - status_msg = f"Saved (+{len(to_add)}, -{len(to_del)})" - if refreshed: - status_msg += f"; loaded {len(tags_to_show)} tag(s)" - try: - app.call_from_thread(self._set_status, status_msg) - except Exception: - self._set_status(status_msg) - except Exception as exc: - try: - app.call_from_thread( - self._set_status, - f"Error: {type(exc).__name__}: {exc}" - ) - except Exception: - self._set_status(f"Error: {type(exc).__name__}: {exc}") - - -class PipelineHubApp(App): - """Textual front-end that executes cmdlet pipelines inline.""" - - CSS_PATH = str(TUI_DIR / "tui.tcss") - BINDINGS = [ - Binding("ctrl+enter", - "run_pipeline", - "Run Pipeline"), - Binding("ctrl+s", - "save_inline_tags", - "Save Tags", - show=False), - Binding("f5", - "refresh_workers", - "Refresh Workers"), - Binding("ctrl+l", - "focus_command", - "Focus Input", - show=False), - Binding("ctrl+g", - "focus_logs", - "Focus Logs", - show=False), - ] - - def __init__(self) -> None: - super().__init__() - self.executor = PipelineRunner() - self.result_items: List[Any] = [] - self.log_lines: List[str] = [] - self.command_input: Optional[Input] = None - self.store_select: Optional[Select] = None - self.path_input: Optional[Input] = None - self.log_output: Optional[TextArea] = None - self.results_table: Optional[DataTable] = None - self.worker_table: Optional[DataTable] = None - self.status_panel: Optional[Static] = None - self.current_result_table: Optional[Table] = None - self.inline_tags_output: Optional[TextArea] = None - self.metadata_tree: Optional[Tree[Any]] = None - self.suggestion_list: Optional[OptionList] = None - self._cmdlet_names: List[str] = [] - self._inline_autocomplete_enabled = False - self._inline_tags_original: List[str] = [] - self._inline_tags_store: str = "" - self._inline_tags_hash: str = "" - self._inline_tags_subject: Any = None - self._pending_pipeline_tags: List[str] = [] - self._pending_pipeline_tags_applied: bool = False - self._pipeline_running = False - self._pipeline_worker: Any = None - self._keep_results_for_current_run: bool = False - self._selected_row_index: int = 0 - self._last_row_select_index: int = -1 - self._last_row_select_at: float = 0.0 - self._zt_server_proc: Optional[subprocess.Popen] = None - self._zt_server_last_config: Optional[str] = None - - # ------------------------------------------------------------------ - # Layout - # ------------------------------------------------------------------ - def compose(self) -> ComposeResult: # noqa: D401 - Textual compose hook - yield Header(show_clock=True) - with Container(id="app-shell"): - with Vertical(id="command-pane"): - with Horizontal(id="command-row"): - yield Input( - placeholder="Enter pipeline command...", - id="pipeline-input" - ) - yield Button("Run", id="run-button") - yield Button("Actions", id="actions-button") - yield Button("Tags", id="tags-button") - yield Button("Metadata", id="metadata-button") - yield Button("Relationships", id="relationships-button") - yield Button("Config", id="config-button") - yield Static("Ready", id="status-panel") - - with Vertical(id="results-pane"): - with Horizontal(id="results-layout"): - with Vertical(id="results-list-pane"): - yield Label("Items", classes="section-title") - yield DataTable(id="results-table") - with Vertical(id="results-tags-pane"): - yield Label("Tags", classes="section-title") - yield TextArea(id="inline-tags-output") - with Vertical(id="results-meta-pane"): - yield Label("Metadata", classes="section-title") - yield Tree("Metadata", id="metadata-tree") - - with Vertical(id="bottom-pane"): - yield Label("Store + Output", classes="section-title") - with Horizontal(id="store-row"): - yield Select([], id="store-select") - yield Input(placeholder="Output path (optional)", id="output-path") - - with Horizontal(id="logs-workers-row"): - with Vertical(id="logs-pane"): - yield Label("Logs", classes="section-title") - yield TextArea(id="log-output", read_only=True) - - with Vertical(id="workers-pane"): - yield Label("Workers", classes="section-title") - yield DataTable(id="workers-table") - yield Footer() - - def on_mount(self) -> None: - self.command_input = self.query_one("#pipeline-input", Input) - self.status_panel = self.query_one("#status-panel", Static) - self.results_table = self.query_one("#results-table", DataTable) - self.worker_table = self.query_one("#workers-table", DataTable) - self.log_output = self.query_one("#log-output", TextArea) - self.store_select = self.query_one("#store-select", Select) - self.path_input = self.query_one("#output-path", Input) - try: - self.suggestion_list = self.query_one("#cmd-suggestions", OptionList) - except Exception: - self.suggestion_list = None - self.inline_tags_output = self.query_one("#inline-tags-output", TextArea) - self.metadata_tree = self.query_one("#metadata-tree", Tree) - - if self.suggestion_list: - self.suggestion_list.display = False - - if self.results_table: - self.results_table.cursor_type = "row" - self.results_table.zebra_stripes = False - try: - self.results_table.cell_padding = 0 - except Exception: - pass - self.results_table.add_columns("Row", "Title", "Source", "File") - if self.worker_table: - self.worker_table.add_columns("ID", "Type", "Status", "Details") - - if self.inline_tags_output: - self.inline_tags_output.text = "" - - if self.metadata_tree: - try: - self.metadata_tree.root.label = "Metadata" - self.metadata_tree.root.remove_children() - self.metadata_tree.root.add("Select an item to view metadata") - self.metadata_tree.root.expand() - except Exception: - pass - - # Initialize the store choices cache at startup (filters disabled stores) - try: - from SYS.cmdlet_spec import SharedArgs - config = load_config() - SharedArgs._refresh_store_choices_cache(config) - except Exception: - logger.exception("Failed to refresh store choices cache") - - self._populate_store_options() - self._load_cmdlet_names() - self._configure_inline_autocomplete() - if self.executor.worker_manager: - self.set_interval(2.0, self.refresh_workers) - self.refresh_workers() - if self.command_input: - self.command_input.focus() - - # Run startup check automatically - self._run_pipeline_background(".status") - - # Provide a visible startup summary of configured plugins/stores for debugging - try: - cfg = load_config() or {} - plugin_block = cfg.get("plugin") - provs = list(plugin_block.keys()) if isinstance(plugin_block, dict) else [] - prov_display = ", ".join(provs[:10]) + ("..." if len(provs) > 10 else "") - self._append_log_line(f"Startup config: plugins={len(provs)} ({prov_display or '(none)'}), db={db.db_path.name}") - except Exception: - logger.exception("Failed to produce startup config summary") - - def on_unmount(self) -> None: - pass - - # ------------------------------------------------------------------ - # Actions - # ------------------------------------------------------------------ - def action_focus_command(self) -> None: - if self.command_input: - self.command_input.focus() - - def action_focus_logs(self) -> None: - if self.log_output: - self.log_output.focus() - - def action_run_pipeline(self) -> None: - if self._pipeline_running: - # Self-heal if the background worker already stopped (e.g. error in thread). - worker = self._pipeline_worker - try: - is_running = bool(getattr(worker, "is_running", False)) - except Exception: - is_running = True - if (worker is None) or (not is_running): - self._pipeline_running = False - self._pipeline_worker = None - else: - self.notify("Pipeline already running", severity="warning", timeout=3) - return - if not self.command_input: - return - pipeline_text = self.command_input.value.strip() - if not pipeline_text: - self.notify("Enter a pipeline to run", severity="warning", timeout=3) - return - - # Special interception for .config - if pipeline_text.lower().strip() == ".config": - self._open_config_popup() - self.command_input.value = "" - return - - pipeline_text = self._apply_store_path_and_tags(pipeline_text) - pipeline_text = self._apply_pending_pipeline_tags(pipeline_text) - self._start_pipeline_execution(pipeline_text) - - def action_save_inline_tags(self) -> None: - if self._pipeline_running: - self.notify("Pipeline already running", severity="warning", timeout=3) - return - - editor = self.inline_tags_output - if editor is None or not bool(getattr(editor, "has_focus", False)): - return - - store_name = str(self._inline_tags_store or "").strip() - file_hash = str(self._inline_tags_hash or "").strip() - seeds = self._inline_tags_subject - selected_item = self._item_for_row_index(self._selected_row_index) - - item, store_name_fallback, file_hash_fallback = self._resolve_selected_item() - if not seeds: - seeds = item - if not store_name and store_name_fallback: - store_name = str(store_name_fallback) - if not file_hash and file_hash_fallback: - file_hash = str(file_hash_fallback) - file_hash = self._normalize_hash_text(file_hash) - - raw_text = "" - try: - raw_text = str(editor.text or "") - except Exception: - raw_text = "" - - desired = _dedup_preserve_order( - [ - str(line).strip() - for line in raw_text.replace("\r\n", "\n").split("\n") - if str(line).strip() - ] - ) - - # Contextual draft mode: no selected result context yet (e.g., pre-download tagging). - if selected_item is None and not file_hash: - self._pending_pipeline_tags = list(desired) - self._pending_pipeline_tags_applied = False - self._inline_tags_original = list(desired) - self._inline_tags_store = str(self._get_selected_store() or "") - self._inline_tags_hash = "" - self._inline_tags_subject = None - self._set_inline_tags(list(desired)) - self._set_status("Saved pending pipeline tags", level="success") - self.notify(f"Saved {len(desired)} pending tag(s)", timeout=3) - return - - if selected_item is not None and not file_hash: - self.notify( - "Selected item is missing a usable hash; cannot save tags to store", - severity="warning", - timeout=5, - ) - return - - if not store_name: - self.notify("Selected item missing store", severity="warning", timeout=4) - return - - current = _dedup_preserve_order(list(self._inline_tags_original or [])) - desired_set = {t.lower() for t in desired} - current_set = {t.lower() for t in current} - to_add = [t for t in desired if t.lower() not in current_set] - to_del = [t for t in current if t.lower() not in desired_set] - - if not to_add and not to_del: - self.notify("No tag changes", timeout=2) - return - - self._set_status("Saving tags…", level="info") - self._save_inline_tags_background( - store_name=store_name, - file_hash=file_hash, - seeds=seeds, - to_add=to_add, - to_del=to_del, - desired=desired, - ) - - def _start_pipeline_execution( - self, - pipeline_text: str, - *, - seeds: Optional[Any] = None, - seed_table: Optional[Any] = None, - clear_log: bool = True, - clear_results: bool = True, - keep_existing_results: bool = False, - ) -> None: - command = str(pipeline_text or "").strip() - if not command: - self.notify("Empty pipeline", severity="warning", timeout=3) - return - - self._pipeline_running = True - self._keep_results_for_current_run = bool(keep_existing_results) - self._set_status("Running…", level="info") - if self.suggestion_list: - try: - self.suggestion_list.display = False - self.suggestion_list.clear_options() # type: ignore[attr-defined] - except Exception: - pass - if clear_log: - self._clear_log() - self._append_log_line(f"$ {command}") - if clear_results: - self._clear_results() - self._pipeline_worker = self._run_pipeline_background(command, seeds, seed_table) - - @on(Input.Changed, "#pipeline-input") - def on_pipeline_input_changed(self, event: Input.Changed) -> None: - text = str(event.value or "") - self._update_suggestions(text) - self._update_syntax_status(text) - - @on(OptionList.OptionSelected, "#cmd-suggestions") - def on_suggestion_selected(self, event: OptionList.OptionSelected) -> None: - if not self.command_input or not self.suggestion_list: - return - try: - suggestion = str(event.option.prompt) - except Exception: - return - new_text = self._apply_suggestion_to_text( - str(self.command_input.value or ""), - suggestion - ) - self.command_input.value = new_text - self._move_command_cursor_to_end() - self.suggestion_list.display = False - self.command_input.focus() - - def action_refresh_workers(self) -> None: - self.refresh_workers() - - # ------------------------------------------------------------------ - # Event handlers - # ------------------------------------------------------------------ - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "run-button": - self.action_run_pipeline() - elif event.button.id == "actions-button": - self._open_actions_popup() - elif event.button.id == "tags-button": - self._open_tags_popup() - elif event.button.id == "metadata-button": - self._open_metadata_popup() - elif event.button.id == "relationships-button": - self._open_relationships_popup() - elif event.button.id == "config-button": - self._open_config_popup() - - def _open_config_popup(self) -> None: - from TUI.modalscreen.config_modal import ConfigModal - self.push_screen(ConfigModal(), callback=self.on_config_closed) - - def on_config_closed(self, result: Any = None) -> None: - """Call when the config modal is dismissed to reload session data.""" - try: - from SYS.config import load_config, clear_config_cache - from SYS.cmdlet_spec import SharedArgs - # Force a fresh load from disk - clear_config_cache() - cfg = load_config() - - # Clear UI state to show a "fresh" start - self._clear_results() - self._clear_log() - self._append_log_line(">>> RESTARTING SESSION (Config updated)") - self._set_status("Reloading config…", level="info") - - # Clear shared caches (especially store selection choices) - SharedArgs._refresh_store_choices_cache(cfg) - # Update the global SharedArgs choices so cmdlets pick up new stores - SharedArgs.STORE.choices = SharedArgs.get_store_choices(cfg, force=True) - - # Re-build our local dropdown - self._populate_store_options() - # Reload cmdlet names (in case new ones were added or indexed) - self._load_cmdlet_names(force=True) - # Optionally update executor config if needed - cfg_loader = getattr(self.executor, "_config_loader", None) - if cfg_loader is not None and hasattr(cfg_loader, "load"): - cfg_loader.load() - - self.notify("Configuration reloaded") - - # Use the existing background runner to show the status table - # This will append the IGNITIO table to the logs/results - self._run_pipeline_background(".status") - - except Exception as exc: - self.notify(f"Error refreshing config: {exc}", severity="error") - - def on_input_submitted(self, event: Input.Submitted) -> None: - if event.input.id == "pipeline-input": - if self.suggestion_list: - try: - self.suggestion_list.display = False - except Exception: - pass - self.action_run_pipeline() - - def on_key(self, event: Key) -> None: - # Make Tab accept autocomplete when typing commands. - if event.key != "tab": - return - if not self.command_input or not self.command_input.has_focus: - return - suggestion = self._get_first_suggestion() - if not suggestion: - suggestion = self._best_cmdlet_match( - self._current_cmd_prefix(str(self.command_input.value or "")) - ) - if not suggestion: - return - - self.command_input.value = self._apply_suggestion_to_text( - str(self.command_input.value or ""), - suggestion - ) - self._move_command_cursor_to_end() - if self.suggestion_list: - self.suggestion_list.display = False - event.prevent_default() - event.stop() - - def _get_first_suggestion(self) -> str: - if not self.suggestion_list or not bool(getattr(self.suggestion_list, - "display", - False)): - return "" - # Textual OptionList API differs across versions; handle best-effort. - try: - options = list(getattr(self.suggestion_list, "options", []) or []) - if options: - first = options[0] - return str(getattr(first, "prompt", "") or "") - except Exception: - logger.exception("Error retrieving first suggestion from suggestion list") - return "" - - def _move_command_cursor_to_end(self) -> None: - if not self.command_input: - return - value = str(self.command_input.value or "") - end_pos = len(value) - - try: - self.command_input.cursor_position = end_pos - return - except Exception: - pass - - try: - self.command_input.cursor_pos = end_pos # type: ignore[attr-defined] - return - except Exception: - pass - - for method_name in ("action_end", "action_cursor_end", "end"): - method = getattr(self.command_input, method_name, None) - if callable(method): - try: - method() - return - except Exception: - continue - - def _best_cmdlet_match(self, prefix: str) -> str: - pfx = str(prefix or "").strip().lower() - if not pfx: - return "" - for name in self._cmdlet_names: - try: - candidate = str(name) - except Exception: - continue - if candidate.lower().startswith(pfx): - return candidate - return "" - - def _configure_inline_autocomplete(self) -> None: - self._inline_autocomplete_enabled = False - if not self.command_input: - return - if SuggestFromList is None: - return - try: - self.command_input.suggester = SuggestFromList( - list(self._cmdlet_names), - case_sensitive=False, - ) - self._inline_autocomplete_enabled = True - except Exception: - self._inline_autocomplete_enabled = False - - def _populate_store_options(self) -> None: - """Populate the store dropdown from the configured Store registry.""" - if not self.store_select: - return - try: - cfg = load_config() or {} - except Exception: - cfg = {} - - stores: List[str] = [] - try: - stores = BackendRegistry(config=cfg, suppress_debug=True).list_backends() - except Exception: - logger.exception("Failed to list store backends from BackendRegistry") - stores = [] - - # Always offer a reasonable default even if config is missing. - if "local" not in [s.lower() for s in stores]: - stores = ["local", *stores] - - options = [(name, name) for name in stores] - try: - self.store_select.set_options(options) - if options: - current = getattr(self.store_select, "value", None) - # Textual Select uses a sentinel for "no selection". - if (current is None) or (current == "") or (current is Select.BLANK): - self.store_select.value = options[0][1] - except Exception: - logger.exception("Failed to set store select options") - - def _get_selected_store(self) -> Optional[str]: - if not self.store_select: - return None - try: - value = getattr(self.store_select, "value", None) - except Exception: - return None - - if value is None or value is Select.BLANK: - return None - try: - text = str(value).strip() - except Exception: - return None - - if not text or text == "Select.BLANK": - return None - return text - - def _apply_store_path_and_tags(self, pipeline_text: str) -> str: - """Apply store/path/tags UI fields to the pipeline text. - - Rules (simple + non-destructive): - - If output path is set and the first stage is file-download and has no -path/--path, append -path. - - If an instance is selected and pipeline has no file-add stage, append file -add -instance . - """ - base = str(pipeline_text or "").strip() - if not base: - return base - - selected_store = self._get_selected_store() - - output_path = "" - if self.path_input: - try: - output_path = str(self.path_input.value or "").strip() - except Exception: - output_path = "" - - stages = [s.strip() for s in base.split("|") if s.strip()] - if not stages: - return base - - # Identify first stage command name for conservative auto-augmentation. - first_stage_cmd = "" - try: - first_stage_cmd = ( - str(stages[0].split()[0]).replace("_", - "-").strip().lower() - if stages[0].split() else "" - ) - except Exception: - first_stage_cmd = "" - - # Apply -path to file-download first stage (only if missing) - if output_path: - first = stages[0] - low = first.lower() - if (low.startswith("download-file") or (low.startswith("file ") and " -download" in low)) and " -path" not in low and " --path" not in low: - stages[0] = f"{first} -path {json.dumps(output_path)}" - - joined = " | ".join(stages) - - low_joined = joined.lower() - - # Only auto-append file -add for download pipelines. - should_auto_add_file = bool( - selected_store and ("add-file" not in low_joined and "file -add" not in low_joined) and ( - first_stage_cmd in {"download-file"} - or (first_stage_cmd == "file" and " -download" in stages[0].lower()) - ) - ) - - if should_auto_add_file: - store_token = json.dumps(selected_store) - joined = f"{joined} | file -add -instance {store_token}" - - return joined - - def _apply_pending_pipeline_tags(self, pipeline_text: str) -> str: - command = str(pipeline_text or "").strip() - pending = list(self._pending_pipeline_tags or []) - if not command or not pending: - self._pending_pipeline_tags_applied = False - return command - - low = command.lower() - if "metadata -add" in low: - # User already controls metadata tag stage explicitly. - self._pending_pipeline_tags_applied = False - return command - - # Apply draft tags when pipeline stores/emits files via file-add. - if "add-file" not in low and "file -add" not in low: - self._pending_pipeline_tags_applied = False - return command - - tag_args = " ".join(json.dumps(t) for t in pending if str(t).strip()) - if not tag_args: - self._pending_pipeline_tags_applied = False - return command - - self._pending_pipeline_tags_applied = True - self.notify(f"Applying {len(pending)} pending tag(s) after pipeline", timeout=3) - return f"{command} | metadata -add {tag_args}" - - def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None: - if not self.results_table or event.control is not self.results_table: - return - index = int(event.cursor_row or 0) - if index < 0: - index = 0 - self._selected_row_index = index - self._refresh_inline_detail_panels(index) - - def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None: - if not self.results_table or event.control is not self.results_table: - return - index = int(event.cursor_row or 0) - if index < 0: - index = 0 - self._selected_row_index = index - self._refresh_inline_detail_panels(index) - if self._is_probable_row_double_click(index): - self._handle_row_double_click(index) - - def _is_probable_row_double_click(self, index: int) -> bool: - now = time.monotonic() - same_row = (int(index) == int(self._last_row_select_index)) - close_in_time = (now - float(self._last_row_select_at)) <= 0.45 - is_double = bool(same_row and close_in_time) - self._last_row_select_index = int(index) - self._last_row_select_at = now - return is_double - - def _handle_row_double_click(self, index: int) -> None: - item = self._item_for_row_index(index) - if item is None: - return - - metadata = self._normalize_item_metadata(item) - table_hint = self._extract_table_hint(metadata) - if not self._is_tidal_artist_context(metadata, table_hint=table_hint): - return - - seed_items = self._current_seed_items() - if not seed_items: - self.notify("No current result items available for artist selection", severity="warning", timeout=3) - return - - row_num = int(index or 0) + 1 - if row_num < 1: - row_num = 1 - - # Mirror CLI behavior: selecting artist row runs @N and opens album list. - self._start_pipeline_execution( - f"@{row_num}", - seeds=seed_items, - seed_table=self.current_result_table, - clear_log=False, - clear_results=False, - keep_existing_results=False, - ) - - # ------------------------------------------------------------------ - # Pipeline execution helpers - # ------------------------------------------------------------------ - @work(exclusive=True, thread=True) - def _run_pipeline_background( - self, - pipeline_text: str, - seeds: Optional[Any] = None, - seed_table: Optional[Any] = None, - ) -> None: - try: - run_result = self.executor.run_pipeline( - pipeline_text, - seeds=seeds, - seed_table=seed_table, - on_log=self._log_from_worker - ) - except Exception as exc: - # Ensure the UI never gets stuck in "running" state. - run_result = PipelineRunResult( - pipeline=str(pipeline_text or ""), - success=False, - error=f"{type(exc).__name__}: {exc}", - stderr=f"{type(exc).__name__}: {exc}", - ) - self.call_from_thread(self._on_pipeline_finished, run_result) - - def _on_pipeline_finished(self, run_result: PipelineRunResult) -> None: - self._pipeline_running = False - self._pipeline_worker = None - keep_existing_results = bool(self._keep_results_for_current_run) - self._keep_results_for_current_run = False - pending_applied = bool(self._pending_pipeline_tags_applied) - pending_count = len(self._pending_pipeline_tags) - if self.suggestion_list: - try: - self.suggestion_list.display = False - self.suggestion_list.clear_options() # type: ignore[attr-defined] - except Exception: - pass - status_level = "success" if run_result.success else "error" - status_text = "Completed" if run_result.success else "Failed" - self._set_status(status_text, level=status_level) - - if not run_result.success: - self.notify( - run_result.error or "Pipeline failed", - severity="error", - timeout=6 - ) - if pending_applied and pending_count: - self.notify("Pending tags were retained (pipeline failed)", severity="warning", timeout=4) - else: - if pending_applied and pending_count: - self._pending_pipeline_tags = [] - self.notify(f"Pipeline completed; applied {pending_count} pending tag(s)", timeout=4) - else: - self.notify("Pipeline completed", timeout=3) - - self._pending_pipeline_tags_applied = False - - if run_result.stdout.strip(): - self._append_log_line("stdout:") - self._append_block(run_result.stdout) - if run_result.stderr.strip(): - self._append_log_line("stderr:") - self._append_block(run_result.stderr) - - for stage in run_result.stages: - summary = f"[{stage.status}] {stage.name} -> {len(stage.emitted)} item(s)" - if stage.error: - summary += f" ({stage.error})" - self._append_log_line(summary) - - if not keep_existing_results: - emitted = run_result.emitted - if isinstance(emitted, list): - self.result_items = emitted - elif emitted: - self.result_items = [emitted] - else: - self.result_items = [] - - self.current_result_table = run_result.result_table - self._selected_row_index = 0 - self._populate_results_table() - self.refresh_workers() - - def _current_seed_items(self) -> List[Any]: - if self.current_result_table and getattr(self.current_result_table, "rows", None): - items: List[Any] = [] - for idx in range(len(self.current_result_table.rows)): - item = self._item_for_row_index(idx) - if item is not None: - items.append(item) - if items: - return items - return list(self.result_items or []) - - def _log_from_worker(self, message: str) -> None: - self.call_from_thread(self._append_log_line, message) - - # ------------------------------------------------------------------ - # UI helpers - # ------------------------------------------------------------------ - @staticmethod - def _extract_title_for_item(item: Any) -> str: - if isinstance(item, dict): - for key in ("title", "name", "path", "url", "hash"): - value = item.get(key) - if value is not None: - text = str(value).strip() - if text: - return text - metadata = item.get("metadata") - if isinstance(metadata, dict): - for key in ("title", "name"): - value = metadata.get(key) - if value is not None: - text = str(value).strip() - if text: - return text - return str(item) - try: - for key in ("title", "name", "path", "url", "hash"): - value = getattr(item, key, None) - if value is not None: - text = str(value).strip() - if text: - return text - except Exception: - pass - return str(item) - - def _item_for_row_index(self, index: int) -> Any: - idx = int(index or 0) - if idx < 0: - return None - - if self.current_result_table and 0 <= idx < len(getattr(self.current_result_table, "rows", []) or []): - row = self.current_result_table.rows[idx] - payload = getattr(row, "payload", None) - if payload is not None: - return payload - src_idx = getattr(row, "source_index", None) - if isinstance(src_idx, int) and 0 <= src_idx < len(self.result_items): - return self.result_items[src_idx] - - if 0 <= idx < len(self.result_items): - return self.result_items[idx] - - return None - - @staticmethod - def _split_tag_text(raw: Any) -> List[str]: - text = str(raw or "").strip() - if not text: - return [] - if "\n" in text or "," in text: - out = [] - for part in re.split(r"[\n,]", text): - p = str(part or "").strip() - if p: - out.append(p) - return out - return [text] - - @staticmethod - def _normalize_hash_text(raw_hash: Any) -> str: - value = str(raw_hash or "").strip().lower() - if len(value) == 64 and all(ch in "0123456789abcdef" for ch in value): - return value - return "" - - def _extract_hash_from_nested(self, value: Any) -> str: - target_keys = {"hash", "hash_hex", "file_hash", "sha256"} - - def _scan(node: Any, depth: int = 0) -> str: - if depth > 8: - return "" - if isinstance(node, dict): - for key in target_keys: - if key in node: - normalized = self._normalize_hash_text(node.get(key)) - if normalized: - return normalized - for child in node.values(): - found = _scan(child, depth + 1) - if found: - return found - return "" - if isinstance(node, (list, tuple, set)): - for child in node: - found = _scan(child, depth + 1) - if found: - return found - return "" - - return _scan(value) - - def _fetch_store_tags(self, store_name: str, file_hash: str) -> Optional[List[str]]: - store_key = str(store_name or "").strip() - hash_key = self._normalize_hash_text(file_hash) - if not store_key or not hash_key: - return None - - try: - cfg = load_config() or {} - except Exception: - cfg = {} - - try: - registry = BackendRegistry(config=cfg, suppress_debug=True) - except Exception: - return None - - match = None - normalized_store = store_key.lower() - for name in registry.list_backends(): - if str(name or "").strip().lower() == normalized_store: - match = name - break - - if match is None: - return None - - try: - backend = registry[match] - except KeyError: - return None - - try: - tags, _src = backend.get_tag(hash_key, config=cfg) - if not tags: - return [] - filtered = [str(t).strip() for t in tags if str(t).strip()] - return _dedup_preserve_order(filtered) - except Exception: - return None - - def _extract_tags_from_nested(self, value: Any) -> List[str]: - tags: List[str] = [] - - def _add_tag(candidate: Any) -> None: - if candidate is None: - return - if isinstance(candidate, (list, tuple, set)): - for entry in candidate: - _add_tag(entry) - return - if isinstance(candidate, str): - tags.extend(self._split_tag_text(candidate)) - return - text = str(candidate).strip() - if text: - tags.append(text) - - def _walk(node: Any) -> None: - if isinstance(node, dict): - for key, child in node.items(): - k = str(key or "").strip().lower() - if k in {"tag", "tags"}: - _add_tag(child) - _walk(child) - return - if isinstance(node, (list, tuple, set)): - for child in node: - _walk(child) - - _walk(value) - return _dedup_preserve_order(tags) - - @staticmethod - def _normalize_item_metadata(item: Any) -> Dict[str, Any]: - if isinstance(item, dict): - data: Dict[str, Any] = {} - for key, value in item.items(): - k = str(key or "").strip() - if k in {"columns", "_selection_args", "_selection_action"}: - continue - data[k] = value - return data - - try: - as_dict = getattr(item, "to_dict", None) - if callable(as_dict): - value = as_dict() - if isinstance(value, dict): - return dict(value) - except Exception: - pass - - return {"value": str(item)} - - @staticmethod - def _collect_metadata_keys(value: Any, out: Optional[set[str]] = None, depth: int = 0) -> set[str]: - target = out if out is not None else set() - if depth > 10: - return target - if isinstance(value, dict): - for key, child in value.items(): - key_text = str(key or "").strip().lower() - if key_text: - target.add(key_text) - PipelineHubApp._collect_metadata_keys(child, target, depth + 1) - return target - if isinstance(value, (list, tuple, set)): - for child in value: - PipelineHubApp._collect_metadata_keys(child, target, depth + 1) - return target - - @staticmethod - def _extract_nested_value_for_keys(value: Any, target_keys: set[str], depth: int = 0) -> str: - if depth > 10: - return "" - if isinstance(value, dict): - for key, child in value.items(): - key_text = str(key or "").strip().lower() - if key_text in target_keys: - val_text = str(child or "").strip() - if val_text: - return val_text - nested = PipelineHubApp._extract_nested_value_for_keys(child, target_keys, depth + 1) - if nested: - return nested - return "" - if isinstance(value, (list, tuple, set)): - for child in value: - nested = PipelineHubApp._extract_nested_value_for_keys(child, target_keys, depth + 1) - if nested: - return nested - return "" - - def _extract_table_hint(self, metadata: Dict[str, Any]) -> str: - hint = self._extract_nested_value_for_keys(metadata, {"table", "source_table", "table_name"}) - if hint: - return str(hint).strip().lower() - try: - current = self.current_result_table - table_attr = str(getattr(current, "table", "") or "").strip().lower() if current else "" - if table_attr: - return table_attr - except Exception: - pass - return "" - - def _is_youtube_context(self, metadata: Dict[str, Any], *, table_hint: str, url_value: str) -> bool: - if "youtube" in str(table_hint or ""): - return True - if isinstance(url_value, str): - low_url = url_value.lower() - if ("youtube.com" in low_url) or ("youtu.be" in low_url): - return True - keys = self._collect_metadata_keys(metadata) - return any("youtube" in key for key in keys) - - def _is_tidal_track_context(self, metadata: Dict[str, Any], *, table_hint: str) -> bool: - table_low = str(table_hint or "").strip().lower() - if ("tidal.track" in table_low) or ("tidal.album" in table_low): - return True - keys = self._collect_metadata_keys(metadata) - return "tidal.track" in keys - - def _is_tidal_artist_context(self, metadata: Dict[str, Any], *, table_hint: str) -> bool: - table_low = str(table_hint or "").strip().lower() - if "tidal.artist" in table_low: - return True - keys = self._collect_metadata_keys(metadata) - has_artist = "tidal.artist" in keys - has_track = "tidal.track" in keys - return bool(has_artist and not has_track) - - def _set_inline_tags(self, tags: List[str]) -> None: - if not self.inline_tags_output: - return - if tags: - self.inline_tags_output.text = "\n".join(tags) - else: - self.inline_tags_output.text = "" - - def _set_metadata_tree(self, metadata: Dict[str, Any]) -> None: - if not self.metadata_tree: - return - try: - root = self.metadata_tree.root - root.label = "Metadata" - root.remove_children() - - def _trim(value: Any) -> str: - text = str(value) - if len(text) > 220: - return text[:217] + "..." - return text - - def _render_node(parent: Any, key: str, value: Any, depth: int = 0) -> None: - if depth > 8: - parent.add(f"{key}: ...") - return - - if isinstance(value, dict): - branch = parent.add(f"{key}") - if not value: - branch.add("{}") - return - for child_key, child_value in value.items(): - _render_node(branch, str(child_key), child_value, depth + 1) - return - - if isinstance(value, (list, tuple, set)): - items = list(value) - branch = parent.add(f"{key} [{len(items)}]") - max_items = 50 - for i, child in enumerate(items[:max_items]): - _render_node(branch, f"[{i}]", child, depth + 1) - if len(items) > max_items: - branch.add(f"... {len(items) - max_items} more") - return - - parent.add(f"{key}: {_trim(value)}") - - if metadata: - for key, value in metadata.items(): - _render_node(root, str(key), value) - else: - root.add("No metadata") - - root.expand() - except Exception: - logger.exception("Failed to render metadata tree") - - def _clear_inline_detail_panels(self) -> None: - pending = list(self._pending_pipeline_tags or []) - self._inline_tags_original = list(pending) - self._inline_tags_store = str(self._get_selected_store() or "") - self._inline_tags_hash = "" - self._inline_tags_subject = None - self._set_inline_tags(pending) - self._set_metadata_tree({}) - - def _refresh_inline_detail_panels(self, index: Optional[int] = None) -> None: - idx = int(self._selected_row_index if index is None else index) - item = self._item_for_row_index(idx) - if item is None: - pending = list(self._pending_pipeline_tags or []) - self._inline_tags_original = list(pending) - self._inline_tags_store = str(self._get_selected_store() or "") - self._inline_tags_hash = "" - self._inline_tags_subject = None - self._set_inline_tags(pending) - self._set_metadata_tree({}) - return - - metadata = self._normalize_item_metadata(item) - tags = self._extract_tags_from_nested(metadata) - _, store_name, file_hash = self._resolve_selected_item() - resolved_hash = self._normalize_hash_text(file_hash) - if not resolved_hash: - resolved_hash = self._extract_hash_from_nested(metadata) - self._inline_tags_original = list(tags) - self._inline_tags_store = str(store_name or "").strip() - self._inline_tags_hash = resolved_hash - self._inline_tags_subject = item - self._set_inline_tags(tags) - self._set_metadata_tree(metadata) - - @work(thread=True) - def _save_inline_tags_background( - self, - *, - store_name: str, - file_hash: str, - seeds: Any, - to_add: List[str], - to_del: List[str], - desired: List[str], - ) -> None: - failures: List[str] = [] - runner = self.executor - store_tok = json.dumps(str(store_name)) - normalized_hash = self._normalize_hash_text(file_hash) - query_chunk = ( - f" -query {json.dumps(f'hash:{normalized_hash}')}" if normalized_hash else "" - ) - - try: - if to_del: - del_args = " ".join(json.dumps(t) for t in to_del) - del_cmd = f"metadata -delete -instance {store_tok}{query_chunk} {del_args}" - del_res = runner.run_pipeline(del_cmd, seeds=seeds, isolate=True) - if not getattr(del_res, "success", False): - failures.append( - str( - getattr(del_res, "error", "") - or getattr(del_res, "stderr", "") - or "metadata -delete failed" - ).strip() - ) - - if to_add: - add_args = " ".join(json.dumps(t) for t in to_add) - add_cmd = f"metadata -add -instance {store_tok}{query_chunk} {add_args}" - add_res = runner.run_pipeline(add_cmd, seeds=seeds, isolate=True) - if not getattr(add_res, "success", False): - failures.append( - str( - getattr(add_res, "error", "") - or getattr(add_res, "stderr", "") - or "metadata -add failed" - ).strip() - ) - - if failures: - msg = failures[0] if failures else "Tag save failed" - self.call_from_thread( - lambda: self._set_status(f"Tag save failed: {msg}", level="error") - ) - self.call_from_thread(self.notify, f"Tag save failed: {msg}", severity="error", timeout=6) - return - - verified_tags = self._fetch_store_tags(store_name, normalized_hash) if normalized_hash else None - if verified_tags is not None: - desired_lower = {str(t).strip().lower() for t in desired if str(t).strip()} - verified_lower = { - str(t).strip().lower() - for t in verified_tags - if str(t).strip() - } - missing = [t for t in desired if str(t).strip().lower() not in verified_lower] - if desired_lower and missing: - preview = ", ".join(missing[:3]) - if len(missing) > 3: - preview += ", ..." - msg = f"Save verification failed; missing tag(s): {preview}" - self.call_from_thread( - lambda: self._set_status(msg, level="error") - ) - self.call_from_thread( - self.notify, - msg, - severity="error", - timeout=6, - ) - return - - final_tags = list(verified_tags) if verified_tags is not None else list(desired) - - def _apply_success() -> None: - self._inline_tags_original = list(final_tags) - self._inline_tags_store = str(store_name or "").strip() - self._inline_tags_hash = normalized_hash - self._inline_tags_subject = seeds - self._set_inline_tags(list(final_tags)) - if normalized_hash: - try: - self.refresh_tag_overlay(store_name, normalized_hash, list(final_tags), seeds) - except Exception: - logger.exception("Failed to refresh tag overlay after inline save") - self._set_status("Tags saved", level="success") - self.notify(f"Saved tags (+{len(to_add)}, -{len(to_del)})", timeout=3) - - self.call_from_thread(_apply_success) - except Exception as exc: - self.call_from_thread( - lambda: self._set_status( - f"Tag save failed: {type(exc).__name__}", - level="error", - ) - ) - self.call_from_thread( - self.notify, - f"Tag save failed: {type(exc).__name__}: {exc}", - severity="error", - timeout=6, - ) - - def _populate_results_table(self) -> None: - if not self.results_table: - return - self.results_table.clear(columns=True) - - def _add_columns_with_widths(headers: List[str], widths: List[int]) -> None: - table = self.results_table - if table is None: - return - for i, header in enumerate(headers): - width = max(1, int(widths[i])) if i < len(widths) else max(1, len(str(header))) - try: - table.add_column(str(header), width=width) - except Exception: - table.add_column(str(header)) - - def _pad_cell(value: Any, width: int, style: str) -> Text: - text = str(value) - if width > 0 and len(text) < width: - text = text + (" " * (width - len(text))) - return Text(text, style=style) - - if self.current_result_table and self.current_result_table.rows: - headers = ["#", "Title"] - normalized_rows: List[List[str]] = [] - - for idx in range(len(self.current_result_table.rows)): - item = self._item_for_row_index(idx) - title_value = self._extract_title_for_item(item) - normalized_rows.append([str(idx + 1), title_value]) - - widths = [len(h) for h in headers] - for row in normalized_rows: - for col_idx, cell in enumerate(row): - cell_len = len(str(cell)) - if cell_len > widths[col_idx]: - widths[col_idx] = cell_len - - _add_columns_with_widths(headers, widths) - - for idx, row in enumerate(normalized_rows, 1): - row_style = get_result_table_row_style(idx - 1) - styled_cells = [ - _pad_cell(cell, widths[col_idx], row_style) - for col_idx, cell in enumerate(row) - ] - self.results_table.add_row(*styled_cells, key=str(idx - 1)) - else: - # Fallback or empty state - headers = ["Row", "Title", "Source", "File"] - if not self.result_items: - self.results_table.add_columns(*headers) - self.results_table.add_row("—", "No results", "", "") - self._clear_inline_detail_panels() - return - - # Fallback for items without a table - raw_rows: List[List[str]] = [] - for idx, item in enumerate(self.result_items, start=1): - raw_rows.append([str(idx), str(item), "—", "—"]) - - widths = [len(h) for h in headers] - for row in raw_rows: - for col_idx, cell in enumerate(row): - cell_len = len(str(cell)) - if cell_len > widths[col_idx]: - widths[col_idx] = cell_len - - _add_columns_with_widths(headers, widths) - - for idx, row in enumerate(raw_rows, start=1): - row_style = get_result_table_row_style(idx - 1) - styled_cells = [ - _pad_cell(cell, widths[col_idx], row_style) - for col_idx, cell in enumerate(row) - ] - self.results_table.add_row(*styled_cells, key=str(idx - 1)) - - if self._item_for_row_index(self._selected_row_index) is None: - self._selected_row_index = 0 - self._refresh_inline_detail_panels(self._selected_row_index) - - def refresh_tag_overlay(self, - store_name: str, - file_hash: str, - tags: List[str], - subject: Any) -> None: - """Update the shared get-tag overlay after manual tag edits.""" - if not store_name or not file_hash: - return - try: - from cmdlet.metadata.tag_get import _emit_tags_as_table - except Exception: - return - - try: - cfg = load_config() or {} - except Exception: - cfg = {} - - payload_subject = subject if subject is not None else None - if not isinstance(payload_subject, dict): - payload_subject = { - "store": store_name, - "hash": file_hash, - } - - try: - _emit_tags_as_table( - list(tags), - file_hash=file_hash, - store=store_name, - config=cfg, - subject=payload_subject, - ) - except Exception: - logger.exception("Failed to emit tags as table") - - def _load_cmdlet_names(self, force: bool = False) -> None: - try: - ensure_registry_loaded(force=force) - names = list_cmdlet_names(force=force) or [] - self._cmdlet_names = sorted( - {str(n).replace("_", - "-") - for n in names if str(n).strip()} - ) - except Exception: - logger.exception("Failed to load cmdlet names") - self._cmdlet_names = [] - self._configure_inline_autocomplete() - - def _update_syntax_status(self, text: str) -> None: - if self._pipeline_running: - return - raw = str(text or "").strip() - if not raw: - self._set_status("Ready", level="info") - return - try: - err = validate_pipeline_text(raw) - except Exception: - err = None - if err: - self._set_status(err.message, level="error") - else: - self._set_status("Ready", level="info") - - def _update_suggestions(self, text: str) -> None: - if self._inline_autocomplete_enabled: - if self.suggestion_list: - try: - self.suggestion_list.display = False - except Exception: - pass - return - if not self.suggestion_list: - return - raw = str(text or "") - prefix = self._current_cmd_prefix(raw) - if not prefix: - self.suggestion_list.display = False - return - - pref_low = prefix.lower() - matches = [n for n in self._cmdlet_names if n.lower().startswith(pref_low)] - matches = matches[:10] - - if not matches: - self.suggestion_list.display = False - return - - try: - self.suggestion_list.clear_options() # type: ignore[attr-defined] - except Exception: - try: - # Fallback for older/newer Textual APIs. - self.suggestion_list.options = [] # type: ignore[attr-defined] - except Exception: - logger.exception("Failed to clear suggestion list options via fallback") - - try: - self.suggestion_list.add_options( - [Option(m) for m in matches] - ) # type: ignore[attr-defined] - except Exception: - try: - self.suggestion_list.options = [ - Option(m) for m in matches - ] # type: ignore[attr-defined] - except Exception: - logger.exception("Failed to set suggestion list options via fallback") - - self.suggestion_list.display = True - - @staticmethod - def _current_cmd_prefix(text: str) -> str: - """Best-effort prefix for cmdlet name completion. - - Completes the token immediately after start-of-line or a '|'. - """ - raw = str(text or "") - # Find the segment after the last pipe. - segment = raw.split("|")[-1] - # Remove leading whitespace. - segment = segment.lstrip() - if not segment: - return "" - # Only complete the first token of the segment. - m = re.match(r"([A-Za-z0-9_\-]*)", segment) - return m.group(1) if m else "" - - @staticmethod - def _apply_suggestion_to_text(text: str, suggestion: str) -> str: - raw = str(text or "") - parts = raw.split("|") - if not parts: - return suggestion - last = parts[-1] - # Preserve leading spaces after the pipe. - leading = "".join(ch for ch in last if ch.isspace()) - trimmed = last.lstrip() - # Replace first token in last segment. - replaced = re.sub(r"^[A-Za-z0-9_\-]*", suggestion, trimmed) - parts[-1] = leading + replaced - return "|".join(parts) - - def _resolve_selected_item( - self - ) -> Tuple[Optional[Any], - Optional[str], - Optional[str]]: - """Return (item, store_name, hash) for the currently selected row.""" - index = int(getattr(self, "_selected_row_index", 0) or 0) - if index < 0: - index = 0 - - item: Any = None - row_payload: Any = None - row = None - column_store: Optional[str] = None - column_hash: Optional[str] = None - - # Prefer mapping displayed table row -> source item. - if self.current_result_table and 0 <= index < len( - getattr(self.current_result_table, - "rows", - []) or []): - row = self.current_result_table.rows[index] - row_payload = getattr(row, "payload", None) - src_idx = getattr(row, "source_index", None) - if isinstance(src_idx, int) and 0 <= src_idx < len(self.result_items): - item = self.result_items[src_idx] - for col in getattr(row, "columns", []) or []: - name = str(getattr(col, "name", "") or "").strip().lower() - value = str(getattr(col, "value", "") or "").strip() - if not column_store and name in {"store", "storage", "source", "table"}: - column_store = value - if not column_hash and name in {"hash", "hash_hex", "file_hash", "sha256"}: - column_hash = value - - if item is None and 0 <= index < len(self.result_items): - item = self.result_items[index] - - def _pick_from_candidates( - candidates: List[Any], extractor: Callable[[Any], str] - ) -> str: - for candidate in candidates: - if candidate is None: - continue - try: - value = extractor(candidate) - except Exception: - value = "" - if value and str(value).strip(): - return str(value).strip() - return "" - - candidate_sources: List[Any] = [] - if row_payload is not None: - candidate_sources.append(row_payload) - if item is not None: - candidate_sources.append(item) - - store_name = _pick_from_candidates(candidate_sources, extract_store_value) - file_hash = _pick_from_candidates(candidate_sources, extract_hash_value) - - if not store_name and column_store: - store_name = column_store - if not file_hash and column_hash: - file_hash = column_hash - - store_text = str(store_name).strip() if store_name else "" - hash_text = str(file_hash).strip() if file_hash else "" - - if not store_text: - # Fallback to UI store selection when item doesn't carry it. - store_text = self._get_selected_store() or "" - - final_item = row_payload if row_payload is not None else item - if final_item is None and (store_text or hash_text): - fallback: Dict[str, str] = {} - if store_text: - fallback["store"] = store_text - if hash_text: - fallback["hash"] = hash_text - final_item = fallback - - return final_item, (store_text or None), (hash_text or None) - - def _open_tags_popup(self) -> None: - if self._pipeline_running: - self.notify("Pipeline already running", severity="warning", timeout=3) - return - item, store_name, file_hash = self._resolve_selected_item() - if item is None: - self.notify("No selected item", severity="warning", timeout=3) - return - if not store_name: - self.notify("Selected item missing store", severity="warning", timeout=4) - return - - seeds: Any = item - if isinstance(item, dict): - seeds = dict(item) - try: - if store_name and not str(seeds.get("store") or "").strip(): - seeds["store"] = store_name - except Exception: - logger.exception("Failed to set seed store value") - try: - if file_hash and not str(seeds.get("hash") or "").strip(): - seeds["hash"] = file_hash - except Exception: - logger.exception("Failed to set seed hash value") - - self.push_screen( - TagEditorPopup(seeds=seeds, - store_name=store_name, - file_hash=file_hash) - ) - - def _open_metadata_popup(self) -> None: - item, _store_name, _file_hash = self._resolve_selected_item() - if item is None: - self.notify("No selected item", severity="warning", timeout=3) - return - text = "" - idx = int(getattr(self, "_selected_row_index", 0) or 0) - if self.current_result_table and 0 <= idx < len( - getattr(self.current_result_table, - "rows", - []) or []): - row = self.current_result_table.rows[idx] - lines = [ - f"{col.name}: {col.value}" for col in getattr(row, "columns", []) or [] - ] - text = "\n".join(lines) - elif isinstance(item, dict): - try: - text = json.dumps(item, indent=2, ensure_ascii=False) - except Exception: - text = str(item) - else: - text = str(item) - self.push_screen(TextPopup(title="Metadata", text=text)) - - def _open_relationships_popup(self) -> None: - item, _store_name, _file_hash = self._resolve_selected_item() - if item is None: - self.notify("No selected item", severity="warning", timeout=3) - return - - relationships = None - if isinstance(item, dict): - relationships = item.get("relationships") or item.get("relationship") - else: - relationships = getattr(item, "relationships", None) - if not relationships: - relationships = getattr(item, "get_relationships", lambda: None)() - - if not relationships: - self.push_screen(TextPopup(title="Relationships", text="No relationships")) - return - - lines: List[str] = [] - if isinstance(relationships, dict): - for rel_type, value in relationships.items(): - if isinstance(value, list): - if not value: - lines.append(f"{rel_type}: (empty)") - for v in value: - lines.append(f"{rel_type}: {v}") - else: - lines.append(f"{rel_type}: {value}") - else: - lines.append(str(relationships)) - self.push_screen(TextPopup(title="Relationships", text="\n".join(lines))) - - def _build_action_context(self) -> Dict[str, Any]: - item, store_name, file_hash = self._resolve_selected_item() - metadata = self._normalize_item_metadata(item) if item is not None else {} - normalized_hash = self._normalize_hash_text(file_hash) - - def _candidate_values() -> List[str]: - out: List[str] = [] - sources: List[Any] = [item, metadata] - for src in sources: - if not isinstance(src, dict): - continue - for key in ("url", "source_url", "target", "path", "file_path"): - raw = src.get(key) - if raw is None: - continue - text = str(raw).strip() - if text: - out.append(text) - return out - - url_value = "" - path_value = "" - for value in _candidate_values(): - low = value.lower() - if (low.startswith("http://") or low.startswith("https://")) and not url_value: - url_value = value - continue - if not path_value: - try: - p = Path(value) - if p.exists(): - path_value = str(p) - except Exception: - pass - - table_hint = self._extract_table_hint(metadata) - is_youtube_context = self._is_youtube_context(metadata, table_hint=table_hint, url_value=url_value) - is_tidal_track_context = self._is_tidal_track_context(metadata, table_hint=table_hint) - is_tidal_artist_context = self._is_tidal_artist_context(metadata, table_hint=table_hint) - can_play_mpv = bool((str(store_name or "").strip() and normalized_hash) or is_youtube_context or is_tidal_track_context) - - return { - "item": item, - "store": str(store_name or "").strip(), - "hash": normalized_hash, - "url": url_value, - "path": path_value, - "selected_store": str(self._get_selected_store() or "").strip(), - "table_hint": table_hint, - "is_youtube_context": is_youtube_context, - "is_tidal_track_context": is_tidal_track_context, - "is_tidal_artist_context": is_tidal_artist_context, - "can_play_mpv": can_play_mpv, - } - - def _build_context_actions(self, ctx_obj: Dict[str, Any]) -> List[Tuple[str, str]]: - actions: List[Tuple[str, str]] = [] - url_value = str(ctx_obj.get("url") or "").strip() - path_value = str(ctx_obj.get("path") or "").strip() - store_name = str(ctx_obj.get("store") or "").strip() - hash_value = str(ctx_obj.get("hash") or "").strip() - selected_store = str(ctx_obj.get("selected_store") or "").strip() - - if url_value: - actions.append(("Open URL", "open_url")) - - if path_value: - actions.append(("Open File", "open_file")) - actions.append(("Open File Folder", "open_folder")) - actions.append(("Copy File Path", "copy_path")) - - can_play_mpv = bool(ctx_obj.get("can_play_mpv", False)) - - if can_play_mpv: - actions.append(("Play in MPV", "play_mpv")) - - if store_name and hash_value: - actions.append(("Delete from Store", "delete_store_item")) - - if selected_store and selected_store.lower() != store_name.lower(): - actions.append( - (f"Copy to Store ({selected_store})", "copy_to_selected_store") - ) - actions.append( - (f"Move to Store ({selected_store})", "move_to_selected_store") - ) - - return actions - - def _open_actions_popup(self) -> None: - if self._pipeline_running: - self.notify("Pipeline already running", severity="warning", timeout=3) - return - - ctx_obj = self._build_action_context() - actions = self._build_context_actions(ctx_obj) - if not actions: - self.notify("No actions available for selected item", severity="warning", timeout=3) - return - - def _run_selected(action_key: Optional[str]) -> None: - if not action_key: - return - self._execute_context_action(str(action_key), ctx_obj) - - self.push_screen(ActionMenuPopup(actions=actions), callback=_run_selected) - - def _copy_text_to_clipboard(self, text: str) -> bool: - content = str(text or "") - if not content: - return False - try: - self.copy_to_clipboard(content) - return True - except Exception: - pass - - try: - import subprocess as _sp - - _sp.run( - [ - "powershell", - "-NoProfile", - "-Command", - f"Set-Clipboard -Value {json.dumps(content)}", - ], - check=False, - capture_output=True, - text=True, - ) - return True - except Exception: - return False - - def _execute_context_action(self, action_key: str, ctx_obj: Dict[str, Any]) -> None: - action = str(action_key or "").strip().lower() - if not action: - return - - url_value = str(ctx_obj.get("url") or "").strip() - path_value = str(ctx_obj.get("path") or "").strip() - store_name = str(ctx_obj.get("store") or "").strip() - hash_value = str(ctx_obj.get("hash") or "").strip() - selected_store = str(ctx_obj.get("selected_store") or "").strip() - - if action == "open_url": - if not url_value: - self.notify("No URL found on selected item", severity="warning", timeout=3) - return - try: - import webbrowser - - webbrowser.open(url_value) - self.notify("Opened URL", timeout=2) - except Exception as exc: - self.notify(f"Failed to open URL: {exc}", severity="error", timeout=4) - return - - if action == "open_file": - if not path_value: - self.notify("No local file path found", severity="warning", timeout=3) - return - try: - p = Path(path_value) - if not p.exists(): - self.notify("Local file path does not exist", severity="warning", timeout=3) - return - if sys.platform.startswith("win"): - os.startfile(str(p)) # type: ignore[attr-defined] - elif sys.platform == "darwin": - subprocess.Popen(["open", str(p)]) - else: - subprocess.Popen(["xdg-open", str(p)]) - self.notify("Opened file", timeout=2) - except Exception as exc: - self.notify(f"Failed to open file: {exc}", severity="error", timeout=4) - return - - if action == "open_folder": - if not path_value: - self.notify("No local file path found", severity="warning", timeout=3) - return - try: - p = Path(path_value) - folder = p.parent if p.is_file() else p - if not folder.exists(): - self.notify("Folder does not exist", severity="warning", timeout=3) - return - if sys.platform.startswith("win"): - os.startfile(str(folder)) # type: ignore[attr-defined] - elif sys.platform == "darwin": - subprocess.Popen(["open", str(folder)]) - else: - subprocess.Popen(["xdg-open", str(folder)]) - self.notify("Opened folder", timeout=2) - except Exception as exc: - self.notify(f"Failed to open folder: {exc}", severity="error", timeout=4) - return - - if action == "copy_path": - if not path_value: - self.notify("No local file path found", severity="warning", timeout=3) - return - if self._copy_text_to_clipboard(path_value): - self.notify("Copied file path", timeout=2) - else: - self.notify("Failed to copy path", severity="error", timeout=3) - return - - if action == "delete_store_item": - if not store_name or not hash_value: - self.notify("Delete action requires store + hash", severity="warning", timeout=3) - return - query = f"hash:{hash_value}" - cmd = f"file -delete -instance {json.dumps(store_name)} -query {json.dumps(query)}" - self._start_pipeline_execution(cmd) - return - - if action == "play_mpv": - if not bool(ctx_obj.get("can_play_mpv", False)): - self.notify("MPV is unavailable for this selected item", severity="warning", timeout=3) - return - selected = self._item_for_row_index(self._selected_row_index) - if selected is None: - self.notify("No selected row for MPV action", severity="warning", timeout=3) - return - seed_items = self._current_seed_items() - if not seed_items: - self.notify("No current result items available for MPV selection", severity="warning", timeout=3) - return - row_num = int(self._selected_row_index or 0) + 1 - if row_num < 1: - row_num = 1 - cmd = f"@{row_num} | .mpv" - self._start_pipeline_execution( - cmd, - seeds=seed_items, - seed_table=self.current_result_table, - clear_log=False, - clear_results=False, - keep_existing_results=True, - ) - return - - if action in {"copy_to_selected_store", "move_to_selected_store"}: - if not store_name or not hash_value or not selected_store: - self.notify("Copy/Move requires source store, hash, and selected target store", severity="warning", timeout=4) - return - if selected_store.lower() == store_name.lower(): - self.notify("Target store must be different from source store", severity="warning", timeout=3) - return - - query = f"hash:{hash_value}" - base_copy = ( - f"file -instance {json.dumps(store_name)} -query {json.dumps(query)}" - f" | file -add -instance {json.dumps(selected_store)}" - ) - if action == "move_to_selected_store": - delete_cmd = f"file -delete -instance {json.dumps(store_name)} -query {json.dumps(query)}" - cmd = f"{base_copy} | @ | {delete_cmd}" - else: - cmd = base_copy - - self._start_pipeline_execution(cmd) - return - - self.notify(f"Unknown action: {action}", severity="warning", timeout=3) - - def _clear_log(self) -> None: - self.log_lines = [] - if self.log_output: - self.log_output.text = "" - - def _append_log_line(self, line: str) -> None: - self.log_lines.append(line) - if len(self.log_lines) > 500: - self.log_lines = self.log_lines[-500:] - if self.log_output: - self.log_output.text = "\n".join(self.log_lines) - - def _append_block(self, text: str) -> None: - for line in text.strip().splitlines(): - self._append_log_line(f" {line}") - - def _clear_results(self) -> None: - self.result_items = [] - if self.results_table: - self.results_table.clear() - self._selected_row_index = 0 - self._clear_inline_detail_panels() - - def _set_status(self, message: str, *, level: str = "info") -> None: - if not self.status_panel: - return - for css in ("status-info", "status-success", "status-error"): - self.status_panel.remove_class(css) - css_class = f"status-{level if level in {'success', 'error'} else 'info'}" - self.status_panel.add_class(css_class) - self.status_panel.update(message) - - def refresh_workers(self) -> None: - if not self.worker_table: - return - manager = self.executor.worker_manager - self.worker_table.clear() - if manager is None: - self.worker_table.add_row("—", "—", "—", "Worker manager unavailable") - return - workers = manager.get_active_workers() - if not workers: - self.worker_table.add_row("—", "—", "—", "No active workers") - return - for worker in workers: - worker_id = str(worker.get("worker_id") or worker.get("id") or "?")[:8] - worker_type = str(worker.get("worker_type") or worker.get("type") or "?") - status = str(worker.get("status") or worker.get("result") or "running") - details = ( - worker.get("current_step") or worker.get("description") - or worker.get("pipe") or "" - ) - self.worker_table.add_row(worker_id, worker_type, status, str(details)[:80]) - - -if __name__ == "__main__": - PipelineHubApp().run() diff --git a/cmdlet/_display.py b/cmdlet/_display.py new file mode 100644 index 0000000..a4ee989 --- /dev/null +++ b/cmdlet/_display.py @@ -0,0 +1,470 @@ +"""UI/display helpers for Rich output, saved-output panels, and result persistence. + +Contains utilities for: +- Printing to stderr safely without disrupting Rich Live display +- Rendering saved output panels when ``-path`` is used +- Persisting items for ``@N`` selection across command boundaries +- Formatting byte sizes for display +- Moving emitted temp/PATH artifacts to user-specified destinations +""" + +from __future__ import annotations + +import shutil +import sys +from contextlib import AbstractContextManager, nullcontext +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from SYS.logger import log +from SYS.result_publication import publish_result_table +from SYS.result_table import Table +from SYS.rich_display import stderr_console as get_stderr_console +from ._pipeobject_utils import get_field, get_pipe_object_path + +__all__ = [ + "_print_live_safe_stderr", + "_print_saved_output_panel", + "_apply_saved_path_update", + "_unique_destination_path", + "_extract_flag_value", + "apply_output_path_from_pipeobjects", + "display_and_persist_items", + "fmt_bytes", +] + + +def fmt_bytes(n: Optional[int]) -> str: + """Format bytes as human-readable with 1 decimal place (MB/GB). + + Args: + n: Number of bytes, or None + + Returns: + Formatted string like "1.5 MB" or "2.0 GB", or "unknown" + """ + if n is None or n < 0: + return "unknown" + gb = n / (1024.0 * 1024.0 * 1024.0) + if gb >= 1.0: + return f"{gb:.1f} GB" + mb = n / (1024.0 * 1024.0) + return f"{mb:.1f} MB" + + +def _extract_flag_value(args: Sequence[str], *flags: str) -> Optional[str]: + """Return the value for the first matching flag in args. + + This is intentionally lightweight (no cmdlet spec required) so callers in CLI/pipeline + can share the same behavior. + """ + if not args: + return None + want = {str(f).strip().lower() + for f in flags if str(f).strip()} + if not want: + return None + try: + tokens = [str(a) for a in args] + except Exception: + tokens = list(args) # type: ignore[list-item] + for i, tok in enumerate(tokens): + low = str(tok).strip().lower() + if low in want: + if i + 1 >= len(tokens): + return None + nxt = str(tokens[i + 1]) + if not nxt.strip(): + return None + if nxt.startswith("-"): + return None + return nxt + return None + + +def _unique_destination_path(dest: Path) -> Path: + """Generate a non-colliding destination path by appending " (N)".""" + try: + if not dest.exists(): + return dest + except Exception: + return dest + + parent = dest.parent + stem = dest.stem + suffix = dest.suffix + for i in range(1, 10_000): + candidate = parent / f"{stem} ({i}){suffix}" + try: + if not candidate.exists(): + return candidate + except Exception: + return candidate + return dest + + +def _print_live_safe_stderr(message: str) -> None: + """Print to stderr without breaking Rich Live progress output.""" + try: + from SYS.rich_display import stderr_console # type: ignore + except Exception: + return + + cm: AbstractContextManager[Any] | None = None + try: + from SYS import pipeline as _pipeline_ctx # type: ignore + + suspend = getattr(_pipeline_ctx, "suspend_live_progress", None) + candidate = suspend() if callable(suspend) else None + if isinstance(candidate, AbstractContextManager): + cm = candidate + elif candidate is not None and hasattr(candidate, "__enter__") and hasattr(candidate, "__exit__"): + cm = candidate # type: ignore[arg-type] + except Exception: + cm = None + + if cm is None: + cm = nullcontext() + + try: + console = stderr_console() + print_func = getattr(console, "print", None) + except Exception: + return + + if not callable(print_func): + return + + try: + with cm: + print_func(str(message)) + except Exception: + return + + +def _print_saved_output_panel(item: Any, final_path: Path) -> None: + """When -path is used, print a Rich panel summarizing the saved output. + + Shows: Title, Location, Hash. + Best-effort: reads existing fields first to avoid recomputing hashes. + """ + try: + from rich.panel import Panel # type: ignore + from rich.table import Table as RichTable # type: ignore + from SYS.rich_display import stderr_console # type: ignore + except Exception: + return + + cm: AbstractContextManager[Any] | None = None + try: + from SYS import pipeline as _pipeline_ctx # type: ignore + + suspend = getattr(_pipeline_ctx, "suspend_live_progress", None) + cm_candidate = suspend() if callable(suspend) else None + if isinstance(cm_candidate, AbstractContextManager): + cm = cm_candidate + elif cm_candidate is not None and hasattr(cm_candidate, "__enter__") and hasattr(cm_candidate, "__exit__"): + cm = cm_candidate # type: ignore[arg-type] + except Exception: + cm = None + + if cm is None: + cm = nullcontext() + + try: + location = str(final_path) + except Exception: + location = "" + + title = "" + try: + title = str(get_field(item, "title") or get_field(item, "name") or "").strip() + except Exception: + title = "" + if not title: + try: + title = str(final_path.stem or final_path.name) + except Exception: + title = "" + + file_hash = "" + try: + file_hash = str(get_field(item, "hash") or get_field(item, "sha256") or "").strip() + except Exception: + file_hash = "" + if not file_hash: + try: + from SYS.utils import sha256_file # type: ignore + + file_hash = str(sha256_file(final_path) or "").strip() + except Exception: + file_hash = "" + + grid = RichTable.grid(padding=(0, 1)) + grid.add_column(justify="right", style="bold") + grid.add_column() + grid.add_row("Title", title or "(unknown)") + grid.add_row("Location", location or "(unknown)") + grid.add_row("Hash", file_hash or "(unknown)") + + try: + console = stderr_console() + print_func = getattr(console, "print", None) + except Exception: + return + + if not callable(print_func): + return + + try: + with cm: + print_func(Panel(grid, title="Saved", expand=False)) + except Exception: + return + + +def _apply_saved_path_update(item: Any, *, old_path: str, new_path: str) -> None: + """Update a PipeObject-like item after its backing file has moved.""" + old_str = str(old_path) + new_str = str(new_path) + if isinstance(item, dict): + try: + if str(item.get("path") or "") == old_str: + item["path"] = new_str + except Exception: + pass + try: + if str(item.get("target") or "") == old_str: + item["target"] = new_str + except Exception: + pass + try: + extra = item.get("extra") + if isinstance(extra, dict): + if str(extra.get("target") or "") == old_str: + extra["target"] = new_str + if str(extra.get("path") or "") == old_str: + extra["path"] = new_str + except Exception: + pass + return + + try: + if getattr(item, "path", None) == old_str: + setattr(item, "path", new_str) + except Exception: + pass + try: + extra = getattr(item, "extra", None) + if isinstance(extra, dict): + if str(extra.get("target") or "") == old_str: + extra["target"] = new_str + if str(extra.get("path") or "") == old_str: + extra["path"] = new_str + except Exception: + pass + + +def apply_output_path_from_pipeobjects( + *, + cmd_name: str, + args: Sequence[str], + emits: Sequence[Any], +) -> List[Any]: + """If the user supplied `-path`, move emitted temp/PATH files there. + + This enables a dynamic pattern: + - Any cmdlet can include `SharedArgs.PATH`. + - If it emits a file-backed PipeObject (`path` exists on disk) and the item is + a temp/PATH artifact, then `-path ` will save it to that location. + + Rules: + - Only affects items whose `action` matches the current cmdlet. + - Only affects items that look like local artifacts (`is_temp` True or `store` == PATH). + - Updates the emitted object's `path` (and `target` when it points at the same file). + """ + dest_raw = _extract_flag_value(args, "-path", "--path") + if not dest_raw: + return list(emits or []) + + try: + dest_str = str(dest_raw).strip() + if "://" in dest_str: + _print_live_safe_stderr( + f"Ignoring -path value that looks like a URL: {dest_str}" + ) + return list(emits or []) + except Exception: + pass + + cmd_norm = str(cmd_name or "").replace("_", "-").strip().lower() + if not cmd_norm: + return list(emits or []) + + try: + dest_hint_dir = str(dest_raw).endswith(("/", "\\")) + except Exception: + dest_hint_dir = False + + try: + dest_path = Path(str(dest_raw)).expanduser() + except Exception: + return list(emits or []) + + items = list(emits or []) + artifact_indices: List[int] = [] + artifact_paths: List[Path] = [] + for idx, item in enumerate(items): + action = str(get_field(item, "action", "") or "").strip().lower() + if not action.startswith("cmdlet:"): + continue + action_name = action.split(":", 1)[-1].strip().lower() + if action_name != cmd_norm: + continue + + store = str(get_field(item, "store", "") or "").strip().lower() + is_temp = bool(get_field(item, "is_temp", False)) + if not (is_temp or store == "path"): + continue + + src_str = get_pipe_object_path(item) + if not src_str: + continue + try: + src = Path(str(src_str)).expanduser() + except Exception: + continue + try: + if not src.exists() or not src.is_file(): + continue + except Exception: + continue + + artifact_indices.append(idx) + artifact_paths.append(src) + + if not artifact_indices: + return items + + if len(artifact_indices) > 1: + if dest_path.suffix: + dest_dir = dest_path.parent + else: + dest_dir = dest_path + try: + dest_dir.mkdir(parents=True, exist_ok=True) + except Exception as exc: + _print_live_safe_stderr( + f"Failed to create destination directory: {dest_dir} ({exc})" + ) + return items + + for idx, src in zip(artifact_indices, artifact_paths): + final = dest_dir / src.name + final = _unique_destination_path(final) + try: + if src.resolve() == final.resolve(): + _apply_saved_path_update( + items[idx], + old_path=str(src), + new_path=str(final), + ) + _print_saved_output_panel(items[idx], final) + continue + except Exception: + pass + try: + shutil.move(str(src), str(final)) + except Exception as exc: + _print_live_safe_stderr(f"Failed to save output to {final}: {exc}") + continue + _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final)) + _print_saved_output_panel(items[idx], final) + + return items + + src = artifact_paths[0] + idx = artifact_indices[0] + final: Path + try: + if dest_hint_dir or (dest_path.exists() and dest_path.is_dir()): + final = dest_path / src.name + else: + final = dest_path + except Exception: + final = dest_path + + try: + final.parent.mkdir(parents=True, exist_ok=True) + except Exception as exc: + _print_live_safe_stderr( + f"Failed to create destination directory: {final.parent} ({exc})" + ) + return items + + final = _unique_destination_path(final) + try: + if src.resolve() != final.resolve(): + shutil.move(str(src), str(final)) + except Exception as exc: + _print_live_safe_stderr(f"Failed to save output to {final}: {exc}") + return items + + _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final)) + _print_saved_output_panel(items[idx], final) + return items + + +def display_and_persist_items( + items: List[Any], + title: str = "Result", + subject: Optional[Any] = None, + display_type: str = "item", + table: Optional[Table] = None, +) -> None: + """Display detail panels and persist items for @N selection. + + This helper function: + 1. Renders individual detail panels for each item + 2. Creates a result table with all items (or uses provided table) + 3. Persists the table so @N selection works across command boundaries + + Args: + items: List of items/dicts to display and make selectable + title: Title for the result table (default: "Result") + subject: Optional subject to associate with the results (usually first item or list) + display_type: Type of display - "item" (default), "tag", or "custom" (no panels) + table: Optional pre-built Table object (if None, creates new Table with title) + """ + if not items: + return + + try: + from SYS import pipeline as pipeline_context + + if table is None: + table = Table(title=title) + + if display_type != "custom": + try: + from SYS.rich_display import render_item_details_panel + + panel_prefix = "Tag" if display_type == "tag" else "Item" + + for idx, item in enumerate(items, 1): + try: + render_item_details_panel(item, title=f"#{idx} {panel_prefix} Details") + except Exception: + pass + except Exception: + pass + + if not getattr(table, "_items_added", False): + for item in items: + table.add_result(item) + + setattr(table, "_rendered_by_cmdlet", True) + + publish_result_table(pipeline_context, table, items, subject=subject) + except Exception: + pass diff --git a/cmdlet/_pipeobject_utils.py b/cmdlet/_pipeobject_utils.py new file mode 100644 index 0000000..0cfd307 --- /dev/null +++ b/cmdlet/_pipeobject_utils.py @@ -0,0 +1,686 @@ +"""PipeObject utilities for chainable cmdlet and multi-action pipelines. + +Contains helpers for creating, normalizing, mutating, and extracting data from +PipeObject instances and result dicts. Also includes the canonical +``merge_sequences`` dedup utility used across cmdlets. +""" + +from __future__ import annotations + +from collections.abc import Iterable as IterableABC +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from SYS import models +from SYS.item_accessors import get_field as _item_accessor_get_field +from SYS.payload_builders import build_file_result_payload +from ._store_utils import resolve_hash_for_cmdlet + +__all__ = [ + "get_field", + "merge_sequences", + "normalize_result_input", + "normalize_result_items", + "value_has_content", + "filter_results_by_temp", + "create_pipe_object_result", + "mark_as_temp", + "set_parent_hash", + "get_pipe_object_path", + "get_pipe_object_hash", + "coerce_to_pipe_object", + "resolve_item_store_hash", + "propagate_metadata", + "extract_relationships", + "extract_duration", + "pipeline_item_local_path", +] + + +def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any: + """Extract a field from either a dict or object with fallback default. + + Handles both dict.get(field) and getattr(obj, field) access patterns. + Also handles lists by accessing the first element. + For PipeObjects, checks the extra field as well. + Used throughout cmdlet to uniformly access fields from mixed types. + + Args: + obj: Dict, object, or list to extract from + field: Field name to retrieve + default: Value to return if field not found (default: None) + + Returns: + Field value if found, otherwise the default value + + Examples: + get_field(result, "hash") # From dict or object + get_field(result, "table", "unknown") # With default + """ + return _item_accessor_get_field(obj, field, default) + + +def merge_sequences(*sources: Optional[Iterable[Any]], + case_sensitive: bool = True) -> list[str]: + """Merge iterable sources while preserving order and removing duplicates.""" + seen: set[str] = set() + merged: list[str] = [] + for source in sources: + if not source: + continue + if isinstance(source, str) or not isinstance(source, IterableABC): + iterable = [source] + else: + iterable = source + for value in iterable: + if value is None: + continue + text = str(value).strip() + if not text: + continue + key = text if case_sensitive else text.lower() + if key in seen: + continue + seen.add(key) + merged.append(text) + return merged + + +def normalize_result_input(result: Any) -> List[Dict[str, Any]]: + """Normalize input result to a list of dicts. + + Handles: + - None -> [] + - Dict -> [dict] + - List of dicts -> list as-is + - PipeObject -> [dict] + - List of PipeObjects -> list of dicts + + Args: + result: Result from piped input + + Returns: + List of result dicts (may be empty) + """ + if result is None: + return [] + + if isinstance(result, dict): + return [result] + + if isinstance(result, list): + output = [] + for item in result: + if isinstance(item, dict): + output.append(item) + elif hasattr(item, "to_dict"): + output.append(item.to_dict()) + else: + output.append(item) + return output + + if hasattr(result, "to_dict"): + return [result.to_dict()] + + if isinstance(result, dict): + return [result] + + return [] + + +def normalize_result_items( + result: Any, + *, + include_falsey_single: bool = False, +) -> List[Any]: + """Normalize piped input to a raw item list without converting item types.""" + if isinstance(result, list): + return list(result) + if result is None: + return [] + if include_falsey_single or result: + return [result] + return [] + + +def value_has_content(value: Any) -> bool: + """Return True when a value should be treated as present for payload building.""" + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, (list, tuple, set)): + return len(value) > 0 + return True + + +def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]: + """Filter results by temporary status. + + Args: + results: List of result dicts or PipeObjects + include_temp: If True, keep temp files; if False, exclude them + + Returns: + Filtered list + """ + if include_temp: + return results + + filtered = [] + for result in results: + is_temp = False + + if hasattr(result, "is_temp"): + is_temp = result.is_temp + elif isinstance(result, dict): + is_temp = result.get("is_temp", False) + + if not is_temp: + filtered.append(result) + + return filtered + + +def create_pipe_object_result( + source: str, + identifier: str, + file_path: str, + cmdlet_name: str, + title: Optional[str] = None, + hash_value: Optional[str] = None, + is_temp: bool = False, + parent_hash: Optional[str] = None, + tag: Optional[List[str]] = None, + **extra: Any, +) -> Dict[str, Any]: + """Create a PipeObject-compatible result dict for pipeline chaining. + + This is a helper to emit results in the standard format that downstream + cmdlet can process (filter, tag, cleanup, etc.). + + Args: + source: Source system (e.g., 'local', 'hydrus', 'download') + identifier: Unique ID from source + file_path: Path to the file + cmdlet_name: Name of the cmdlet that created this (e.g., 'download-data', 'screen-shot') + title: Human-readable title + hash_value: SHA-256 hash of file (for integrity) + is_temp: If True, this is a temporary/intermediate artifact + parent_hash: Hash of the parent file in the chain (for provenance) + tag: List of tag values to apply + **extra: Additional fields + + Returns: + Dict with all PipeObject fields for emission + """ + store_override = extra.pop("store", None) + + result = build_file_result_payload( + title=title, + path=file_path, + hash_value=hash_value, + store=store_override if store_override is not None else source, + tag=tag, + source=source, + id=identifier, + action=f"cmdlet:{cmdlet_name}", + **extra, + ) + + if is_temp: + result["is_temp"] = True + if parent_hash: + result["parent_hash"] = parent_hash + + return result + + +def mark_as_temp(pipe_object: Dict[str, Any]) -> Dict[str, Any]: + """Mark a PipeObject dict as temporary (intermediate artifact). + + Args: + pipe_object: Result dict from cmdlet emission + + Returns: + Modified dict with is_temp=True + """ + pipe_object["is_temp"] = True + return pipe_object + + +def set_parent_hash(pipe_object: Dict[str, Any], parent_hash: str) -> Dict[str, Any]: + """Set the parent_hash for provenance tracking. + + Args: + pipe_object: Result dict + parent_hash: Parent file's hash + + Returns: + Modified dict with parent_hash set to the hash + """ + pipe_object["parent_hash"] = parent_hash + return pipe_object + + +def get_pipe_object_path(pipe_object: Any) -> Optional[str]: + """Extract file path from PipeObject, dict, or pipeline-friendly object.""" + if pipe_object is None: + return None + for attr in ("path", "target"): + if hasattr(pipe_object, attr): + value = getattr(pipe_object, attr) + if value: + return value + if isinstance(pipe_object, dict): + for key in ("path", "target"): + value = pipe_object.get(key) + if value: + return value + return None + + +def get_pipe_object_hash(pipe_object: Any) -> Optional[str]: + """Extract file hash from PipeObject, dict, or pipeline-friendly object.""" + if pipe_object is None: + return None + for attr in ("hash",): + if hasattr(pipe_object, attr): + value = getattr(pipe_object, attr) + if value: + return value + if isinstance(pipe_object, dict): + for key in ("hash",): + value = pipe_object.get(key) + if value: + return value + return None + + +def resolve_item_store_hash( + item: Any, + *, + override_store: Optional[str] = None, + override_hash: Optional[str] = None, + hash_field: str = "hash", + store_field: str = "store", + path_fields: Sequence[str] = ("path", "target"), +) -> Tuple[str, Optional[str]]: + """Resolve store name and normalized hash from a result item.""" + store_name = str(override_store or get_field(item, store_field) or "").strip() + raw_hash = get_field(item, hash_field) + + raw_path = None + for field_name in path_fields: + candidate = get_field(item, field_name) + if candidate: + raw_path = candidate + break + + resolved_hash = resolve_hash_for_cmdlet( + str(raw_hash) if raw_hash else None, + str(raw_path) if raw_path else None, + str(override_hash) if override_hash else None, + ) + return store_name, resolved_hash + + +def coerce_to_pipe_object( + value: Any, + default_path: Optional[str] = None, +) -> models.PipeObject: + """Normalize any incoming result to a PipeObject for single-source-of-truth state. + + Uses hash+store canonical pattern. + """ + if isinstance(value, models.PipeObject): + return value + + known_keys = { + "hash", + "store", + "tag", + "title", + "url", + "source_url", + "duration", + "metadata", + "warnings", + "path", + "relationships", + "is_temp", + "action", + "parent_hash", + } + + if hasattr(value, "to_dict"): + value = value.to_dict() + elif not isinstance(value, dict): + try: + obj_map: Dict[str, Any] = {} + for k in ( + "hash", + "store", + "provider", + "prov", + "tag", + "title", + "url", + "source_url", + "duration", + "duration_seconds", + "metadata", + "full_metadata", + "warnings", + "path", + "target", + "relationships", + "is_temp", + "action", + "parent_hash", + "extra", + "media_kind", + ): + if hasattr(value, k): + obj_map[k] = getattr(value, k) + if obj_map: + value = obj_map + except Exception: + pass + + if isinstance(value, dict): + hash_val = value.get("hash") + store_val = value.get("store") or "PATH" + if not store_val or store_val == "PATH": + try: + extra_store = value.get("extra", {}).get("store") + except Exception: + extra_store = None + if extra_store: + store_val = extra_store + + if not hash_val: + path_val = value.get("path") + if path_val: + try: + from SYS.utils import sha256_file + + hash_val = sha256_file(Path(path_val)) + except Exception: + hash_val = "unknown" + else: + hash_val = "unknown" + + title_val = value.get("title") + if not title_val: + path_val = value.get("path") + if path_val: + try: + title_val = Path(path_val).stem + except Exception: + pass + + extra = { + k: v + for k, v in value.items() if k not in known_keys + } + + from SYS.metadata import normalize_urls + + url_list = normalize_urls(value.get("url")) + url_val = url_list[0] if url_list else None + if len(url_list) > 1: + extra["url"] = url_list + + rels = value.get("relationships") or {} + + tag_val: list[str] = [] + if "tag" in value: + raw_tag = value["tag"] + if isinstance(raw_tag, list): + tag_val = [str(t) for t in raw_tag if t is not None] + elif isinstance(raw_tag, str): + tag_val = [raw_tag] + + path_val = value.get("path") + if not path_val and "target" in value: + target = value["target"] + if target and not (isinstance(target, str) and (target.startswith("http://") + or target.startswith("https://"))): + path_val = target + + try: + if isinstance(path_val, str) and (path_val.startswith("http://") + or path_val.startswith("https://")): + if not url_val: + url_val = path_val + path_val = None + except Exception: + pass + + if "media_kind" in value: + extra["media_kind"] = value["media_kind"] + + pipe_obj = models.PipeObject( + hash=hash_val, + store=store_val, + plugin=str( + value.get("plugin") + or value.get("prov") + or value.get("source") + or extra.get("plugin") + or extra.get("source") + or "" + ).strip() or None, + tag=tag_val, + title=title_val, + url=url_val, + source_url=value.get("source_url"), + duration=value.get("duration") or value.get("duration_seconds"), + metadata=value.get("metadata") or value.get("full_metadata") or {}, + warnings=list(value.get("warnings") or []), + path=path_val, + relationships=rels, + is_temp=bool(value.get("is_temp", False)), + action=value.get("action"), + parent_hash=value.get("parent_hash"), + extra=extra, + ) + + return pipe_obj + + hash_val = "unknown" + path_val = default_path or getattr(value, "path", None) + url_val: Optional[str] = None + title_val = None + + if isinstance(value, str): + s = value.strip() + if s.lower().startswith(("http://", "https://")): + url_val = s + path_val = None + else: + path_val = s + + if path_val and path_val != "unknown": + try: + from SYS.utils import sha256_file + + path_obj = Path(path_val) + hash_val = sha256_file(path_obj) + title_val = path_obj.stem + except Exception: + pass + + store_val = "URL" if url_val else "PATH" + + pipe_obj = models.PipeObject( + hash=hash_val, + store=store_val, + plugin=None, + path=str(path_val) if path_val and path_val != "unknown" else None, + title=title_val, + url=url_val, + source_url=url_val, + tag=[], + extra={}, + ) + + return pipe_obj + + +def propagate_metadata( + previous_items: Sequence[Any], + new_items: Sequence[Any], +) -> List[Any]: + """Merge metadata/tags from previous pipeline stage into new items. + + Implements "sticky metadata": items generated by a transformation (download, convert) + should inherit rich info (lyrics, art, tags) from their source. + + Strategies: + A. Hash Match: If inputs/outputs share a hash, they are the same item. + B. Index Match: If lists are same length, assume 1:1 mapping (heuristic). + C. Explicit Parent: If output has `parent_hash`, link to input with that hash. + """ + if not previous_items or not new_items: + return list(new_items) + + try: + prev_normalized = [coerce_to_pipe_object(p) for p in previous_items] + except Exception: + return list(new_items) + + prev_by_hash: Dict[str, models.PipeObject] = {} + for p_obj in prev_normalized: + if p_obj.hash and p_obj.hash != "unknown": + prev_by_hash[p_obj.hash] = p_obj + + normalized: List[Any] = [] + + is_same_length = len(new_items) == len(prev_normalized) + + for i, item in enumerate(new_items): + if isinstance(item, dict) and item.get("_skip_metadata_propagation"): + normalized.append(item) + continue + try: + obj = coerce_to_pipe_object(item) + except Exception: + normalized.append(item) + continue + + parent: Optional[models.PipeObject] = None + + if obj.hash in prev_by_hash: + parent = prev_by_hash[obj.hash] + + if not parent and is_same_length: + parent = prev_normalized[i] + + if not parent and obj.parent_hash and obj.parent_hash in prev_by_hash: + parent = prev_by_hash[obj.parent_hash] + + if parent: + if parent.tag: + if not obj.tag: + obj.tag = list(parent.tag) + else: + curr_tags = {str(t).lower() for t in obj.tag} + for pt in parent.tag: + if str(pt).lower() not in curr_tags: + obj.tag.append(pt) + + if parent.metadata: + if not obj.metadata: + obj.metadata = parent.metadata.copy() + else: + for mk, mv in parent.metadata.items(): + if mk not in obj.metadata: + obj.metadata[mk] = mv + + if parent.source_url and not obj.source_url: + obj.source_url = parent.source_url + elif parent.url and not obj.source_url and not obj.url: + obj.source_url = parent.url + + if parent.relationships: + if not obj.relationships: + obj.relationships = parent.relationships.copy() + else: + for rk, rv in parent.relationships.items(): + if rk not in obj.relationships: + obj.relationships[rk] = rv + + if parent.extra: + if not obj.extra: + obj.extra = parent.extra.copy() + else: + for ek, ev in parent.extra.items(): + if ek not in obj.extra: + obj.extra[ek] = ev + elif ek == "notes" and isinstance(ev, dict) and isinstance(obj.extra[ek], dict): + for nk, nv in ev.items(): + if nk not in obj.extra[ek]: + obj.extra[ek][nk] = nv + + normalized.append(obj) + + return normalized + + +def pipeline_item_local_path(item: Any) -> Optional[str]: + """Extract local file path from a pipeline item. + + Supports both dataclass objects with .path attribute and dicts. + Returns None for HTTP/HTTPS url. + + Args: + item: Pipeline item (PipelineItem dataclass, dict, or other) + + Returns: + Local file path string, or None if item is not a local file + """ + path_value: Optional[str] = None + if hasattr(item, "path"): + path_value = getattr(item, "path", None) + elif isinstance(item, dict): + raw = item.get("path") or item.get("url") + path_value = str(raw) if raw is not None else None + if not isinstance(path_value, str): + return None + text = path_value.strip() + if not text: + return None + if text.lower().startswith(("http://", "https://")): + return None + return text + + +def extract_relationships(result: Any) -> Optional[Dict[str, Any]]: + if isinstance(result, models.PipeObject): + relationships = result.get_relationships() + return relationships or None + if isinstance(result, dict): + relationships = result.get("relationships") + if isinstance(relationships, dict) and relationships: + return relationships + return None + + +def extract_duration(result: Any) -> Optional[float]: + duration = None + if isinstance(result, models.PipeObject): + duration = result.duration + elif isinstance(result, dict): + duration = result.get("duration") + if duration is None: + metadata = result.get("metadata") + if isinstance(metadata, dict): + duration = metadata.get("duration") + if duration is None: + return None + try: + return float(duration) + except (TypeError, ValueError): + return None diff --git a/cmdlet/_preflight.py b/cmdlet/_preflight.py new file mode 100644 index 0000000..14a72e4 --- /dev/null +++ b/cmdlet/_preflight.py @@ -0,0 +1,80 @@ +"""Store backend resolution and preflight validation helpers. + +Resolves store backends for cmdlets, preferring targeted instances before +falling back to the backend registry. Used by store-related cmdlets to avoid +repeated registry setup boilerplate. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple + +__all__ = [ + "get_store_backend", + "get_preferred_store_backend", +] + + +def get_store_backend( + config: Optional[Dict[str, Any]], + store_name: Optional[str], + *, + store_registry: Any = None, + suppress_debug: bool = False, +) -> Tuple[Optional[Any], Any, Optional[Exception]]: + """Resolve a store backend, optionally reusing an existing registry. + + Returns a tuple of ``(backend, store_registry, exc)`` so callers can keep + their command-specific error messages while avoiding repeated registry setup + and ``store[name]`` boilerplate. + """ + registry = store_registry + if registry is None: + try: + from PluginCore.backend_registry import BackendRegistry + + registry = BackendRegistry(config or {}, suppress_debug=suppress_debug) + except Exception as exc: + return None, None, exc + + backend_name = str(store_name or "").strip() + if not backend_name: + return None, registry, KeyError("Missing store name") + + try: + return registry[backend_name], registry, None + except Exception as exc: + return None, registry, exc + + +def get_preferred_store_backend( + config: Optional[Dict[str, Any]], + store_name: Optional[str], + *, + store_registry: Any = None, + suppress_debug: bool = True, +) -> Tuple[Optional[Any], Any, Optional[Exception]]: + """Prefer a targeted backend instance before falling back to registry lookup.""" + direct_exc: Optional[Exception] = None + try: + from PluginCore.backend_registry import get_backend_instance + + backend = get_backend_instance( + config or {}, + str(store_name or ""), + suppress_debug=suppress_debug, + ) + if backend is not None: + return backend, store_registry, None + except Exception as exc: + direct_exc = exc + + backend, registry, lookup_exc = get_store_backend( + config, + store_name, + store_registry=store_registry, + suppress_debug=suppress_debug, + ) + if backend is not None: + return backend, registry, None + return None, registry, direct_exc or lookup_exc diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 8062679..01ba571 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -1,53 +1,125 @@ -""" """ +"""Shared utilities for cmdlets — re-export module. + +This module re-exports all utilities from the focused sub-modules to maintain +backward compatibility with existing imports. New code should import directly +from the appropriate sub-module, but existing ``from cmdlet._shared import ...`` +imports continue to work unchanged. + +Sub-modules: +- ``_tag_utils`` — tag parsing, normalization, template rendering, extraction +- ``_pipeobject_utils`` — PipeObject creation, coercion, metadata propagation +- ``_url_utils`` — URL preflight checks, merging, removal +- ``_template_utils`` — pipeline preview and template helpers +- ``_preflight`` — store backend resolution and validation +- ``_display`` — Rich output, saved-panel rendering, result persistence +- ``_store_utils`` — hash resolution, store batch dispatching, Hydrus metadata +""" from __future__ import annotations -import base64 -import hashlib -import json -import re -import shutil -import sys import tempfile -import time -from collections.abc import Iterable as IterableABC -from functools import lru_cache -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence +from contextlib import AbstractContextManager, nullcontext from SYS.logger import log, debug, debug_panel -from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple from SYS import models from SYS import pipeline as pipeline_context -from SYS.item_accessors import get_field as _item_accessor_get_field from SYS.payload_builders import build_file_result_payload, build_table_result_payload from SYS.result_publication import publish_result_table from SYS.result_table import Table from SYS.rich_display import stderr_console as get_stderr_console from SYS.cmdlet_spec import Cmdlet, CmdletArg, QueryArg, SharedArgs, parse_cmdlet_args from rich.prompt import Confirm -from contextlib import AbstractContextManager, nullcontext +from ._tag_utils import ( + set_tag_groups_path, + normalize_hash, + looks_like_hash, + parse_tag_arguments, + render_tag_value_templates, + build_tag_value_lookup, + _add_tag_values_to_lookup, + expand_tag_groups, + first_title_tag, + apply_preferred_title, + collapse_namespace_tags, + collect_relationship_labels, + extract_tag_from_result, + extract_title_from_result, + extract_url_from_result, + TAG_GROUPS_PATH, +) -# Tag groups cache (loaded from JSON config file) -_TAG_GROUPS_CACHE: Optional[Dict[str, List[str]]] = None -_TAG_GROUPS_MTIME: Optional[float] = None +from ._pipeobject_utils import ( + get_field, + merge_sequences, + normalize_result_input, + normalize_result_items, + value_has_content, + filter_results_by_temp, + create_pipe_object_result, + mark_as_temp, + set_parent_hash, + get_pipe_object_path, + get_pipe_object_hash, + coerce_to_pipe_object, + resolve_item_store_hash, + propagate_metadata, + extract_relationships, + extract_duration, + pipeline_item_local_path, +) -# Path to tag groups configuration (set by caller or lazily discovered) -TAG_GROUPS_PATH: Optional[Path] = None +from ._url_utils import ( + check_url_exists_in_storage, + merge_urls, + remove_urls, + set_item_urls, + register_url_with_local_library, +) +from ._template_utils import ( + build_pipeline_preview, +) -def set_tag_groups_path(path: Path) -> None: - """Set the path to the tag groups JSON file.""" - global TAG_GROUPS_PATH - TAG_GROUPS_PATH = path +from ._preflight import ( + get_store_backend, + get_preferred_store_backend, +) + +from ._display import ( + _print_live_safe_stderr, + _print_saved_output_panel, + _apply_saved_path_update, + _unique_destination_path, + _extract_flag_value, + apply_output_path_from_pipeobjects, + display_and_persist_items, + fmt_bytes, +) + +from ._store_utils import ( + resolve_hash_for_cmdlet, + parse_hash_query, + parse_single_hash_query, + require_hash_query, + require_single_hash_query, + get_hash_for_operation, + fetch_hydrus_metadata, + coalesce_hash_value_pairs, + run_store_hash_value_batches, + run_store_note_batches, + collect_store_hash_value_batch, +) def resolve_target_dir( parsed: Dict[str, Any], config: Dict[str, Any], *, - handle_creations: bool = True + handle_creations: bool = True, ) -> Optional[Path]: """Resolve a target directory from -path, -storage, or config fallback. @@ -59,16 +131,17 @@ def resolve_target_dir( Returns: Path to the resolved directory, or None if invalid. """ - # Priority 1: Explicit -path + from SYS.utils import expand_path + target = parsed.get("path") if target: try: from SYS.config import resolve_path_alias - from SYS.utils import expand_path raw_target = str(target or "").strip() aliased = resolve_path_alias(config, raw_target) if raw_target.startswith("$") and aliased is None: + from SYS.logger import log log( f"Unknown path alias {raw_target}. Set it with .config path_aliases. ", file=sys.stderr, @@ -80,19 +153,19 @@ def resolve_target_dir( p.mkdir(parents=True, exist_ok=True) return p except Exception as e: + from SYS.logger import log log(f"Cannot use target path {target}: {e}", file=sys.stderr) return None - # Priority 2: --storage flag storage_val = parsed.get("storage") if storage_val: try: return SharedArgs.resolve_storage(storage_val) except Exception as e: + from SYS.logger import log log(f"Invalid storage location: {e}", file=sys.stderr) return None - # Priority 3: Config fallback via single source of truth try: from SYS.config import resolve_output_dir out_dir = resolve_output_dir(config) @@ -100,7 +173,6 @@ def resolve_target_dir( out_dir.mkdir(parents=True, exist_ok=True) return out_dir except Exception: - import tempfile p = Path(tempfile.gettempdir()) / "Medios-Macina" if handle_creations: p.mkdir(parents=True, exist_ok=True) @@ -114,12 +186,10 @@ def coerce_to_path(value: Any) -> Path: if isinstance(value, str): return Path(value) - # Try attribute p = getattr(value, "path", None) if p: return Path(str(p)) - # Try dict if isinstance(value, dict): p = value.get("path") if p: @@ -128,7 +198,6 @@ def coerce_to_path(value: Any) -> Path: raise ValueError(f"Cannot coerce {type(value).__name__} to Path (missing 'path' field)") - def resolve_media_kind_by_extension(path: Path) -> str: """Resolve media kind (audio, video, image, document, other) from file extension.""" if not isinstance(path, Path): @@ -136,645 +205,19 @@ def resolve_media_kind_by_extension(path: Path) -> str: path = Path(str(path)) except Exception: return "other" - + suffix = path.suffix.lower() - if suffix in {".mp3", - ".flac", - ".wav", - ".m4a", - ".aac", - ".ogg", - ".opus", - ".wma", - ".mka"}: + if suffix in {".mp3", ".flac", ".wav", ".m4a", ".aac", ".ogg", ".opus", ".wma", ".mka"}: return "audio" - if suffix in { - ".mp4", - ".mkv", - ".webm", - ".mov", - ".avi", - ".flv", - ".mpg", - ".mpeg", - ".ts", - ".m4v", - ".wmv", - }: + if suffix in {".mp4", ".mkv", ".webm", ".mov", ".avi", ".flv", ".mpg", ".mpeg", ".ts", ".m4v", ".wmv"}: return "video" - if suffix in {".jpg", - ".jpeg", - ".png", - ".gif", - ".webp", - ".bmp", - ".tiff"}: + if suffix in {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff"}: return "image" - if suffix in {".pdf", - ".epub", - ".txt", - ".mobi", - ".azw3", - ".cbz", - ".cbr", - ".doc", - ".docx"}: + if suffix in {".pdf", ".epub", ".txt", ".mobi", ".azw3", ".cbz", ".cbr", ".doc", ".docx"}: return "document" return "other" -def build_pipeline_preview(raw_urls: Sequence[str], piped_items: Sequence[Any]) -> List[str]: - """Construct a short preview list for pipeline/cmdlet progress UI.""" - preview: List[str] = [] - - # 1. Add raw URLs - try: - for u in (raw_urls or [])[:3]: - if u: - preview.append(str(u)) - except Exception: - pass - - # 2. Add titles from piped items - if len(preview) < 5: - try: - for item in (piped_items or [])[:5]: - if len(preview) >= 5: - break - title = get_field(item, "title") or get_field(item, "target") or "Piped item" - preview.append(str(title)) - except Exception: - pass - - # 3. Handle empty case - if not preview: - total = len(raw_urls or []) + len(piped_items or []) - if total: - preview.append(f"Processing {total} item(s)...") - - return preview - - -@lru_cache(maxsize=4096) -def _normalize_hash_cached(hash_hex: str) -> Optional[str]: - text = hash_hex.strip().lower() - if not text: - return None - if len(text) != 64: - return None - if not all(ch in "0123456789abcdef" for ch in text): - return None - return text - - -def normalize_hash(hash_hex: Optional[str]) -> Optional[str]: - """Normalize a hash string to lowercase, or return None if invalid. - - Args: - hash_hex: String that should be a hex hash - - Returns: - Lowercase hash string, or None if input is not a string or is empty - """ - if not isinstance(hash_hex, str): - return None - return _normalize_hash_cached(hash_hex) - - -def resolve_hash_for_cmdlet( - raw_hash: Optional[str], - raw_path: Optional[str], - override_hash: Optional[str], -) -> Optional[str]: - """Resolve a file hash for note/tag/file cmdlets. - - Shared implementation used by add-note, delete-note, get-note, and similar - cmdlets that need to identify a file by its SHA-256 hash. - - Resolution order: - 1. ``override_hash`` — explicit hash provided via *-query* (highest priority) - 2. ``raw_hash`` — positional hash argument - 3. ``raw_path`` stem — if the filename stem is a 64-char hex string it is - treated directly as the hash (Hydrus-style naming convention) - 4. SHA-256 computed from the file at ``raw_path`` - - Args: - raw_hash: Hash string from positional argument. - raw_path: Filesystem path to the file (may be None). - override_hash: Hash extracted from *-query* (takes precedence). - - Returns: - Normalised 64-char lowercase hex hash, or ``None`` if unresolvable. - """ - resolved = normalize_hash(override_hash) if override_hash else normalize_hash(raw_hash) - if resolved: - return resolved - if raw_path: - try: - p = Path(str(raw_path)) - stem = p.stem - if len(stem) == 64 and all(c in "0123456789abcdef" for c in stem.lower()): - return stem.lower() - if p.exists() and p.is_file(): - from SYS.utils import sha256_file as _sha256_file - return _sha256_file(p) - except Exception: - return None - return None - - -def resolve_item_store_hash( - item: Any, - *, - override_store: Optional[str] = None, - override_hash: Optional[str] = None, - hash_field: str = "hash", - store_field: str = "store", - path_fields: Sequence[str] = ("path", "target"), -) -> Tuple[str, Optional[str]]: - """Resolve store name and normalized hash from a result item.""" - store_name = str(override_store or get_field(item, store_field) or "").strip() - raw_hash = get_field(item, hash_field) - - raw_path = None - for field_name in path_fields: - candidate = get_field(item, field_name) - if candidate: - raw_path = candidate - break - - resolved_hash = resolve_hash_for_cmdlet( - str(raw_hash) if raw_hash else None, - str(raw_path) if raw_path else None, - str(override_hash) if override_hash else None, - ) - return store_name, resolved_hash - - -def get_store_backend( - config: Optional[Dict[str, Any]], - store_name: Optional[str], - *, - store_registry: Any = None, - suppress_debug: bool = False, -) -> Tuple[Optional[Any], Any, Optional[Exception]]: - """Resolve a store backend, optionally reusing an existing registry. - - Returns a tuple of ``(backend, store_registry, exc)`` so callers can keep - their command-specific error messages while avoiding repeated registry setup - and ``store[name]`` boilerplate. - """ - registry = store_registry - if registry is None: - try: - from PluginCore.backend_registry import BackendRegistry - - registry = BackendRegistry(config or {}, suppress_debug=suppress_debug) - except Exception as exc: - return None, None, exc - - backend_name = str(store_name or "").strip() - if not backend_name: - return None, registry, KeyError("Missing store name") - - try: - return registry[backend_name], registry, None - except Exception as exc: - return None, registry, exc - - -def get_preferred_store_backend( - config: Optional[Dict[str, Any]], - store_name: Optional[str], - *, - store_registry: Any = None, - suppress_debug: bool = True, -) -> Tuple[Optional[Any], Any, Optional[Exception]]: - """Prefer a targeted backend instance before falling back to registry lookup.""" - direct_exc: Optional[Exception] = None - try: - from PluginCore.backend_registry import get_backend_instance - - backend = get_backend_instance( - config or {}, - str(store_name or ""), - suppress_debug=suppress_debug, - ) - if backend is not None: - return backend, store_registry, None - except Exception as exc: - direct_exc = exc - - backend, registry, lookup_exc = get_store_backend( - config, - store_name, - store_registry=store_registry, - suppress_debug=suppress_debug, - ) - if backend is not None: - return backend, registry, None - return None, registry, direct_exc or lookup_exc - - -def coalesce_hash_value_pairs( - pairs: Sequence[Tuple[str, Sequence[str]]], -) -> List[Tuple[str, List[str]]]: - """Merge duplicate hash/value pairs while preserving first-seen value order.""" - merged: Dict[str, List[str]] = {} - for hash_value, values in pairs: - normalized_hash = str(hash_value or "").strip() - if not normalized_hash: - continue - bucket = merged.setdefault(normalized_hash, []) - seen = set(bucket) - for value in values or []: - text = str(value or "").strip() - if not text or text in seen: - continue - seen.add(text) - bucket.append(text) - return [(hash_value, items) for hash_value, items in merged.items() if items] - - -def run_store_hash_value_batches( - config: Optional[Dict[str, Any]], - batch: Dict[str, List[Tuple[str, Sequence[str]]]], - *, - bulk_method_name: str, - single_method_name: str, - store_registry: Any = None, - suppress_debug: bool = False, - pass_config_to_bulk: bool = True, - pass_config_to_single: bool = True, -) -> Tuple[Any, List[Tuple[str, int, int]]]: - """Dispatch grouped hash/value batches across stores. - - Returns ``(store_registry, stats)`` where ``stats`` contains - ``(store_name, item_count, value_count)`` for each dispatched store. - Missing stores are skipped so callers can preserve existing warning behavior. - """ - registry = store_registry - stats: List[Tuple[str, int, int]] = [] - for store_name, pairs in batch.items(): - backend, registry, _exc = get_store_backend( - config, - store_name, - store_registry=registry, - suppress_debug=suppress_debug, - ) - if backend is None: - continue - - bulk_pairs = coalesce_hash_value_pairs(pairs) - if not bulk_pairs: - continue - - bulk_fn = getattr(backend, bulk_method_name, None) - if callable(bulk_fn): - if pass_config_to_bulk: - bulk_fn(bulk_pairs, config=config) - else: - bulk_fn(bulk_pairs) - else: - single_fn = getattr(backend, single_method_name) - for hash_value, values in bulk_pairs: - if pass_config_to_single: - single_fn(hash_value, values, config=config) - else: - single_fn(hash_value, values) - - stats.append( - ( - store_name, - len(bulk_pairs), - sum(len(values or []) for _hash_value, values in bulk_pairs), - ) - ) - - return registry, stats - - -def run_store_note_batches( - config: Optional[Dict[str, Any]], - batch: Dict[str, List[Tuple[str, str, str]]], - *, - store_registry: Any = None, - suppress_debug: bool = False, - on_store_error: Optional[Callable[[str, Exception], None]] = None, - on_unsupported_store: Optional[Callable[[str], None]] = None, - on_item_error: Optional[Callable[[str, str, str, Exception], None]] = None, -) -> Tuple[Any, int]: - """Dispatch grouped note writes across stores while preserving item-level errors.""" - registry = store_registry - success_count = 0 - for store_name, items in batch.items(): - backend, registry, exc = get_store_backend( - config, - store_name, - store_registry=registry, - suppress_debug=suppress_debug, - ) - if backend is None: - if on_store_error is not None and exc is not None: - on_store_error(store_name, exc) - continue - supports_note_capability = getattr(backend, "supports_note_association", None) - if supports_note_capability is None: - supports_note_capability = hasattr(backend, "set_note") - if not bool(supports_note_capability): - if on_unsupported_store is not None: - on_unsupported_store(store_name) - continue - if not hasattr(backend, "set_note"): - if on_unsupported_store is not None: - on_unsupported_store(store_name) - continue - - for hash_value, note_name, note_text in items: - try: - if backend.set_note(hash_value, note_name, note_text, config=config): - success_count += 1 - except Exception as item_exc: - if on_item_error is not None: - on_item_error(store_name, hash_value, note_name, item_exc) - - return registry, success_count - - -def collect_store_hash_value_batch( - items: Sequence[Any], - *, - store_registry: Any, - value_resolver: Callable[[Any], Optional[Sequence[str]]], - override_hash: Optional[str] = None, - override_store: Optional[str] = None, - on_warning: Optional[Callable[[str], None]] = None, -) -> Tuple[Dict[str, List[Tuple[str, List[str]]]], List[Any]]: - """Collect validated store/hash/value batches while preserving passthrough items.""" - batch: Dict[str, List[Tuple[str, List[str]]]] = {} - pass_through: List[Any] = [] - - for item in items: - pass_through.append(item) - - raw_hash = override_hash or get_field(item, "hash") - raw_store = override_store or get_field(item, "store") - if not raw_hash or not raw_store: - if on_warning is not None: - on_warning("Item missing hash/store; skipping") - continue - - normalized = normalize_hash(raw_hash) - if not normalized: - if on_warning is not None: - on_warning("Item has invalid hash; skipping") - continue - - store_text = str(raw_store).strip() - if not store_text: - if on_warning is not None: - on_warning("Item has empty store; skipping") - continue - - try: - is_available = bool(store_registry.is_available(store_text)) - except Exception: - is_available = False - if not is_available: - if on_warning is not None: - on_warning(f"Store '{store_text}' not configured; skipping") - continue - - values = [str(value).strip() for value in (value_resolver(item) or []) if str(value).strip()] - if not values: - continue - - batch.setdefault(store_text, []).append((normalized, values)) - - return batch, pass_through - - -def parse_hash_query(query: Optional[str]) -> List[str]: - """Parse a unified query string for `hash:` into normalized SHA256 hashes. - - Supported examples: - - hash:

- - hash:

,

,

- - Hash:

- - hash:{

,

} - - Returns: - List of unique normalized 64-hex SHA256 hashes. - """ - import re - - q = str(query or "").strip() - if not q: - return [] - - m = re.match(r"^hash(?:es)?\s*:\s*(.+)$", q, flags=re.IGNORECASE) - if not m: - return [] - - rest = (m.group(1) or "").strip() - if rest.startswith("{") and rest.endswith("}"): - rest = rest[1:-1].strip() - if rest.startswith("[") and rest.endswith("]"): - rest = rest[1:-1].strip() - - raw_parts = [p.strip() for p in re.split(r"[\s,]+", rest) if p.strip()] - out: List[str] = [] - for part in raw_parts: - h = normalize_hash(part) - if not h: - continue - if h not in out: - out.append(h) - return out - - -def parse_single_hash_query(query: Optional[str]) -> Optional[str]: - """Parse `hash:` query and require exactly one hash.""" - hashes = parse_hash_query(query) - if len(hashes) != 1: - return None - return hashes[0] - - -def require_hash_query( - query: Optional[str], - error_message: str, - *, - log_file: Any = None, -) -> Tuple[List[str], bool]: - """Parse a multi-hash query and log a caller-provided error on invalid input.""" - hashes = parse_hash_query(query) - if query and not hashes: - kwargs = {"file": log_file} if log_file is not None else {} - log(error_message, **kwargs) - return [], False - return hashes, True - - -def require_single_hash_query( - query: Optional[str], - error_message: str, - *, - log_file: Any = None, -) -> Tuple[Optional[str], bool]: - """Parse a single-hash query and log a caller-provided error on invalid input.""" - query_hash = parse_single_hash_query(query) - if query and not query_hash: - kwargs = {"file": log_file} if log_file is not None else {} - log(error_message, **kwargs) - return None, False - return query_hash, True - - -def get_hash_for_operation( - override_hash: Optional[str], - result: Any, - field_name: str = "hash" -) -> Optional[str]: - """Get normalized hash from override or result object, consolidating common pattern. - - Eliminates repeated pattern: normalize_hash(override) if override else normalize_hash(get_field(result, ...)) - - Args: - override_hash: Hash passed as command argument (takes precedence) - result: Object containing hash field (fallback) - field_name: Name of hash field in result object (default: "hash") - - Returns: - Normalized hash string, or None if neither override nor result provides valid hash - """ - if override_hash: - return normalize_hash(override_hash) - hash_value = ( - get_field(result, - field_name) or getattr(result, - field_name, - None) or getattr(result, - "hash", - None) - ) - return normalize_hash(hash_value) - - -def fetch_hydrus_metadata( - config: Any, - hash_hex: str, - *, - store_name: Optional[str] = None, - hydrus_client: Any = None, - **kwargs, -) -> tuple[Optional[Dict[str, - Any]], - Optional[int]]: - """Fetch metadata from Hydrus for a given hash, consolidating common fetch pattern. - - Eliminates repeated boilerplate: client initialization, error handling, metadata extraction. - - Args: - config: Configuration object used to resolve the Hydrus plugin/store - hash_hex: File hash to fetch metadata for - store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client. - hydrus_client: Optional explicit Hydrus client. When provided, takes precedence. - **kwargs: Additional arguments to pass to client.fetch_file_metadata() - Common: include_service_keys_to_tags, include_notes, include_file_url, include_duration, etc. - - Returns: - Tuple of (metadata_dict, error_code) - - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable - - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute()) - """ - client = hydrus_client - hydrus_provider = None - try: - from PluginCore.registry import get_plugin - - hydrus_provider = get_plugin("hydrusnetwork", config) - except Exception: - hydrus_provider = None - - if client is None: - if hydrus_provider is not None: - try: - client = hydrus_provider.get_client( - store_name=store_name if store_name else None, - allow_default=not bool(store_name), - ) - except Exception as exc: - if store_name: - log(f"Hydrus client unavailable for store '{store_name}': {exc}") - else: - log(f"Hydrus client unavailable: {exc}") - client = None - if client is None and store_name: - log(f"Hydrus client unavailable for store '{store_name}'") - return None, 1 - if client is None and hydrus_provider is None: - log("Hydrus provider unavailable") - return None, 1 - - if hydrus_provider is not None: - try: - metadata = hydrus_provider.fetch_metadata( - hash_hex, - store_name=store_name if store_name else None, - **kwargs, - ) - except Exception as exc: - log(f"Hydrus metadata fetch failed: {exc}") - return None, 1 - if isinstance(metadata, dict): - return metadata, 0 - if client is None: - if store_name: - log(f"Hydrus client unavailable for store '{store_name}'") - else: - log("Hydrus metadata unavailable") - return None, 1 - - try: - payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs) - except Exception as exc: - log(f"Hydrus metadata fetch failed: {exc}") - return None, 1 - - items = payload.get("metadata") if isinstance(payload, dict) else None - meta = items[0] if ( - isinstance(items, - list) and items and isinstance(items[0], - dict) - ) else None - - return meta, 0 - - -def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any: - """Extract a field from either a dict or object with fallback default. - - Handles both dict.get(field) and getattr(obj, field) access patterns. - Also handles lists by accessing the first element. - For PipeObjects, checks the extra field as well. - Used throughout cmdlet to uniformly access fields from mixed types. - - Args: - obj: Dict, object, or list to extract from - field: Field name to retrieve - default: Value to return if field not found (default: None) - - Returns: - Field value if found, otherwise the default value - - Examples: - get_field(result, "hash") # From dict or object - get_field(result, "table", "unknown") # With default - """ - return _item_accessor_get_field(obj, field, default) - - def should_show_help(args: Sequence[str]) -> bool: """Check if help flag was passed in arguments. @@ -793,2935 +236,7 @@ def should_show_help(args: Sequence[str]) -> bool: """ try: return any( - str(a).lower() in {"-?", - "/?", - "--help", - "-h", - "help", - "--cmdlet"} for a in args + str(a).lower() in {"-?", "/?", "--help", "-h", "help", "--cmdlet"} for a in args ) except Exception: return False - - -def looks_like_hash(candidate: Optional[str]) -> bool: - """Check if a string looks like a SHA256 hash (64 hex chars). - - Args: - candidate: String to test - - Returns: - True if the string is 64 lowercase hex characters - """ - if not isinstance(candidate, str): - return False - text = candidate.strip().lower() - return len(text) == 64 and all(ch in "0123456789abcdef" for ch in text) - - -def pipeline_item_local_path(item: Any) -> Optional[str]: - """Extract local file path from a pipeline item. - - Supports both dataclass objects with .path attribute and dicts. - Returns None for HTTP/HTTPS url. - - Args: - item: Pipeline item (PipelineItem dataclass, dict, or other) - - Returns: - Local file path string, or None if item is not a local file - """ - path_value: Optional[str] = None - if hasattr(item, "path"): - path_value = getattr(item, "path", None) - elif isinstance(item, dict): - raw = item.get("path") or item.get("url") - path_value = str(raw) if raw is not None else None - if not isinstance(path_value, str): - return None - text = path_value.strip() - if not text: - return None - if text.lower().startswith(("http://", "https://")): - return None - return text - - -def collect_relationship_labels( - payload: Any, - label_stack: List[str] | None = None, - mapping: Dict[str, - str] | None = None -) -> Dict[str, - str]: - """Recursively extract hash-to-label mappings from nested relationship data. - - Walks through nested dicts/lists looking for sha256-like strings (64 hex chars) - and builds a mapping from hash to its path in the structure. - - Example: - data = { - "duplicates": [ - "abc123...", # Will be mapped to "duplicates" - {"type": "related", "items": ["def456..."]} # Will be mapped to "duplicates / type / items" - ] - } - result = collect_relationship_labels(data) - # result = {"abc123...": "duplicates", "def456...": "duplicates / type / items"} - - Args: - payload: Nested data structure (dict, list, string, etc.) - label_stack: Internal use - tracks path during recursion - mapping: Internal use - accumulates hash->label mappings - - Returns: - Dict mapping hash strings to their path labels - """ - if label_stack is None: - label_stack = [] - if mapping is None: - mapping = {} - - if isinstance(payload, dict): - for key, value in payload.items(): - next_stack = label_stack - if isinstance(key, str) and key: - formatted = key.replace("_", " ").strip() - next_stack = label_stack + [formatted] - collect_relationship_labels(value, next_stack, mapping) - elif isinstance(payload, (list, tuple, set)): - for value in payload: - collect_relationship_labels(value, label_stack, mapping) - elif isinstance(payload, str) and looks_like_hash(payload): - hash_value = payload.lower() - if label_stack: - label = " / ".join(item for item in label_stack if item) - else: - label = "related" - mapping.setdefault(hash_value, label) - - return mapping - - -def parse_tag_arguments(arguments: Sequence[str]) -> List[str]: - """Parse tag arguments from command line tokens. - - - Supports comma-separated tags. - - Supports pipe namespace shorthand: "artist:A|B|C" -> artist:A, artist:B, artist:C. - - Args: - arguments: Sequence of argument strings - - Returns: - List of normalized tag strings (empty strings filtered out) - """ - - def _split_top_level_commas(text: str) -> List[str]: - segments: List[str] = [] - current: List[str] = [] - paren_depth = 0 - angle_depth = 0 - quote: Optional[str] = None - escape = False - - for ch in text: - if escape: - current.append(ch) - escape = False - continue - if ch == "\\": - current.append(ch) - escape = True - continue - if quote: - current.append(ch) - if ch == quote: - quote = None - continue - if ch in {"'", '"'}: - current.append(ch) - quote = ch - continue - if ch == "(": - paren_depth += 1 - current.append(ch) - continue - if ch == ")": - paren_depth = max(0, paren_depth - 1) - current.append(ch) - continue - if ch == "<": - angle_depth += 1 - current.append(ch) - continue - if ch == ">": - angle_depth = max(0, angle_depth - 1) - current.append(ch) - continue - if ch == "," and paren_depth == 0 and angle_depth == 0: - segments.append("".join(current).strip()) - current = [] - continue - current.append(ch) - - tail = "".join(current).strip() - if tail or segments: - segments.append(tail) - return segments - - def _expand_pipe_namespace(text: str) -> List[str]: - parts = text.split("|") - expanded: List[str] = [] - last_ns: Optional[str] = None - for part in parts: - segment = part.strip() - if not segment: - continue - if ":" in segment: - ns, val = segment.split(":", 1) - ns = ns.strip() - val = val.strip() - last_ns = ns or last_ns - if last_ns and val: - expanded.append(f"{last_ns}:{val}") - elif ns or val: - expanded.append(f"{ns}:{val}".strip(":")) - else: - if last_ns: - expanded.append(f"{last_ns}:{segment}") - else: - expanded.append(segment) - return expanded - - tags: List[str] = [] - for argument in arguments: - for token in _split_top_level_commas(str(argument)): - text = token.strip() - if not text: - continue - # Expand namespace shorthand with pipes - pipe_expanded = _expand_pipe_namespace(text) - for entry in pipe_expanded: - candidate = entry.strip() - if not candidate: - continue - if ":" in candidate: - ns, val = candidate.split(":", 1) - ns = ns.strip() - val = val.strip() - candidate = f"{ns}:{val}" if ns or val else "" - if candidate: - tags.append(candidate) - return tags - - -_TAG_VALUE_TEMPLATE_RE = re.compile(r"#\(([^)]+)\)") -_TAG_VALUE_FUNCTION_RE = re.compile(r"<([a-zA-Z_][a-zA-Z0-9_-]*)\((.*?)\)>") - - -def _normalize_tag_value_template_name(value: Any) -> str: - text = str(value or "").strip().lower() - if not text: - return "" - try: - text = re.sub(r"\s+", " ", text).strip() - except Exception: - text = " ".join(text.split()) - return text - - -def _tag_value_template_keys(value: Any) -> list[str]: - normalized = _normalize_tag_value_template_name(value) - if not normalized: - return [] - - keys = [normalized] - - trimmed_hash = re.sub(r"\s*#+\s*$", "", normalized).strip() - if trimmed_hash and trimmed_hash not in keys: - keys.append(trimmed_hash) - - return keys - - -def _add_tag_values_to_lookup(lookup: Dict[str, List[str]], tag_text: Any) -> None: - text = str(tag_text or "").strip() - if not text or ":" not in text: - return - if _TAG_VALUE_TEMPLATE_RE.search(text) or _TAG_VALUE_FUNCTION_RE.search(text): - return - - namespace, value = text.split(":", 1) - value_text = str(value or "").strip() - if not value_text: - return - - for key in _tag_value_template_keys(namespace): - values = lookup.setdefault(key, []) - if value_text not in values: - values.append(value_text) - - -def build_tag_value_lookup( - tags: Optional[Iterable[Any]], - *, - result: Any = None, -) -> Dict[str, List[str]]: - """Build a placeholder lookup from existing tags and lightweight result fields. - - Placeholder lookups use ``#(namespace)`` syntax. Namespace matching is - case-insensitive and trims repeated whitespace. A trailing ``#`` in the - placeholder is ignored so inputs like ``#(track #)`` can resolve ``track:9``. - """ - - lookup: Dict[str, List[str]] = {} - for tag in tags or []: - _add_tag_values_to_lookup(lookup, tag) - - title_text = extract_title_from_result(result) - if title_text: - _add_tag_values_to_lookup(lookup, f"title:{title_text}") - - return lookup - - -def _split_tag_value_function_args(value: Any) -> list[str]: - text = str(value or "") - args: list[str] = [] - current: list[str] = [] - depth = 0 - quote: Optional[str] = None - escape = False - - for ch in text: - if escape: - current.append(ch) - escape = False - continue - if ch == "\\": - current.append(ch) - escape = True - continue - if quote: - current.append(ch) - if ch == quote: - quote = None - continue - if ch in {"'", '"'}: - current.append(ch) - quote = ch - continue - if ch in {"(", "[", "{"}: - depth += 1 - current.append(ch) - continue - if ch in {")", "]", "}"}: - depth = max(0, depth - 1) - current.append(ch) - continue - if ch == "," and depth == 0: - args.append("".join(current).strip()) - current = [] - continue - current.append(ch) - - tail = "".join(current).strip() - if tail or args: - args.append(tail) - return args - - -def _strip_tag_value_function_arg(value: Any) -> str: - text = str(value or "").strip() - if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}: - return text[1:-1] - return text - - -def _padding_width_from_spec(value: Any) -> Optional[int]: - spec = _strip_tag_value_function_arg(value) - if not spec: - return None - if re.fullmatch(r"0+", spec): - return len(spec) - if spec.isdigit(): - try: - width = int(spec) - except Exception: - return None - return width if width > 0 else None - return None - - -def _replace_tag_value_placeholders( - value: Any, - lookup: Dict[str, List[str]], - *, - preserve_unresolved: bool, -) -> tuple[str, bool]: - text = str(value or "") - unresolved = False - - def _replace(match: re.Match[str]) -> str: - nonlocal unresolved - keys = _tag_value_template_keys(match.group(1) or "") - values: List[str] = [] - for key in keys: - for candidate in lookup.get(key, []): - if candidate not in values: - values.append(candidate) - if not values: - unresolved = True - return match.group(0) if preserve_unresolved else "" - return ", ".join(values) - - return _TAG_VALUE_TEMPLATE_RE.sub(_replace, text), unresolved - - -def _coerce_tag_value_integer(value: Any) -> Optional[int]: - text = _strip_tag_value_function_arg(value) - if not text: - return None - if not re.fullmatch(r"[+-]?\d+", text): - return None - try: - return int(text) - except Exception: - return None - - -def _apply_tag_value_function( - name: str, - args: Sequence[str], - *, - lookup: Dict[str, List[str]], -) -> Optional[str]: - func = str(name or "").strip().lower() - - resolved_values: list[str] = [] - unresolved_flags: list[bool] = [] - for arg in args: - rendered, unresolved = _replace_tag_value_placeholders( - arg, - lookup, - preserve_unresolved=True, - ) - resolved_values.append(_strip_tag_value_function_arg(rendered)) - unresolved_flags.append(unresolved) - - if func in {"padding", "pad", "zfill"}: - if len(resolved_values) != 2 or any(unresolved_flags): - return None - width = _padding_width_from_spec(resolved_values[0]) - if width is None: - return None - return str(resolved_values[1]).zfill(width) - - if func == "default": - if len(resolved_values) != 2: - return None - primary = resolved_values[0] - fallback = resolved_values[1] - if not unresolved_flags[0] and str(primary).strip(): - return str(primary) - if unresolved_flags[1]: - return None - return str(fallback) - - if func == "replace": - if len(resolved_values) != 3 or any(unresolved_flags): - return None - return str(resolved_values[0]).replace( - str(resolved_values[1]), - str(resolved_values[2]), - ) - - if func in {"increment", "inc", "add"}: - if len(resolved_values) not in {1, 2}: - return None - if unresolved_flags[0]: - return None - base_value = _coerce_tag_value_integer(resolved_values[0]) - if base_value is None: - return None - step_value = 1 - if len(resolved_values) == 2: - if unresolved_flags[1]: - return None - parsed_step = _coerce_tag_value_integer(resolved_values[1]) - if parsed_step is None: - return None - step_value = parsed_step - return str(base_value + step_value) - - return None - - -def _render_tag_value_function_templates( - value: Any, - *, - lookup: Dict[str, List[str]], -) -> tuple[str, bool]: - text = str(value or "") - unresolved = False - - def _replace(match: re.Match[str]) -> str: - nonlocal unresolved - func_name = match.group(1) or "" - func_args = _split_tag_value_function_args(match.group(2) or "") - rendered = _apply_tag_value_function( - func_name, - func_args, - lookup=lookup, - ) - if rendered is None: - unresolved = True - return match.group(0) - return rendered - - previous = None - rendered = text - while previous != rendered and _TAG_VALUE_FUNCTION_RE.search(rendered): - previous = rendered - rendered = _TAG_VALUE_FUNCTION_RE.sub(_replace, rendered) - if unresolved: - break - return rendered, unresolved - - -def render_tag_value_templates( - tags: Sequence[Any], - *, - existing_tags: Optional[Iterable[Any]] = None, - result: Any = None, -) -> tuple[list[str], list[str]]: - """Resolve ``#(namespace)`` placeholders and ```` functions. - - Returns ``(resolved_tags, unresolved_templates)``. Tags whose placeholders - cannot be fully resolved are omitted from ``resolved_tags`` and returned in - ``unresolved_templates`` so callers can warn or summarize skipped items. - - Currently supported transforms: - - ```` or ```` for zero-padding - - ```` to fall back when a placeholder is missing - - ```` for simple substring replacement - - ```` for integer arithmetic - """ - - entries: list[dict[str, Any]] = [] - lookup = build_tag_value_lookup(existing_tags, result=result) - - for raw_tag in tags or []: - text = str(raw_tag or "").strip() - if not text: - continue - has_template = bool( - _TAG_VALUE_TEMPLATE_RE.search(text) - or _TAG_VALUE_FUNCTION_RE.search(text) - ) - entry = { - "raw": text, - "resolved": None, - "has_template": has_template, - } - if not has_template: - entry["resolved"] = text - _add_tag_values_to_lookup(lookup, text) - entries.append(entry) - - progress = True - while progress: - progress = False - for entry in entries: - if entry["resolved"] is not None or not entry["has_template"]: - continue - - rendered, unresolved = _replace_tag_value_placeholders( - entry["raw"], - lookup, - preserve_unresolved=bool(_TAG_VALUE_FUNCTION_RE.search(str(entry["raw"]))), - ) - - rendered, function_unresolved = _render_tag_value_function_templates( - rendered, - lookup=lookup, - ) - if function_unresolved: - continue - - if unresolved and _TAG_VALUE_TEMPLATE_RE.search(rendered): - continue - - rendered = rendered.strip() - if not rendered: - entry["resolved"] = "" - progress = True - continue - - entry["resolved"] = rendered - _add_tag_values_to_lookup(lookup, rendered) - progress = True - - resolved_tags = merge_sequences( - [entry["resolved"] for entry in entries if isinstance(entry.get("resolved"), str) and entry.get("resolved")], - case_sensitive=True, - ) - unresolved_templates = [ - str(entry["raw"]) - for entry in entries - if entry["has_template"] and not entry.get("resolved") - ] - return resolved_tags, unresolved_templates - - -def fmt_bytes(n: Optional[int]) -> str: - """Format bytes as human-readable with 1 decimal place (MB/GB). - - Args: - n: Number of bytes, or None - - Returns: - Formatted string like "1.5 MB" or "2.0 GB", or "unknown" - """ - if n is None or n < 0: - return "unknown" - gb = n / (1024.0 * 1024.0 * 1024.0) - if gb >= 1.0: - return f"{gb:.1f} GB" - mb = n / (1024.0 * 1024.0) - return f"{mb:.1f} MB" - - -def _normalize_tag_group_entry(value: Any) -> Optional[str]: - """Internal: Normalize a single tag group entry.""" - if not isinstance(value, str): - value = str(value) - text = value.strip() - return text or None - - -def _load_tag_groups() -> Dict[str, List[str]]: - """Load tag group definitions from JSON file with caching.""" - global _TAG_GROUPS_CACHE, _TAG_GROUPS_MTIME, TAG_GROUPS_PATH - - # Auto-discover adjective.json if not set - if TAG_GROUPS_PATH is None: - # Try to find adjective.json in the script directory or helper subdirectory - try: - script_dir = Path(__file__).parent.parent - - # Check root directory - candidate = script_dir / "adjective.json" - if candidate.exists(): - TAG_GROUPS_PATH = candidate - else: - # Check helper directory - candidate = script_dir / "helper" / "adjective.json" - if candidate.exists(): - TAG_GROUPS_PATH = candidate - except Exception: - pass - - if TAG_GROUPS_PATH is None: - return {} - - path = TAG_GROUPS_PATH - try: - stat_result = path.stat() - except FileNotFoundError: - _TAG_GROUPS_CACHE = {} - _TAG_GROUPS_MTIME = None - return {} - except OSError as exc: - log(f"Failed to read tag groups: {exc}", file=sys.stderr) - _TAG_GROUPS_CACHE = {} - _TAG_GROUPS_MTIME = None - return {} - - mtime = stat_result.st_mtime - if _TAG_GROUPS_CACHE is not None and _TAG_GROUPS_MTIME == mtime: - return _TAG_GROUPS_CACHE - - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - log(f"Invalid tag group JSON ({path}): {exc}", file=sys.stderr) - _TAG_GROUPS_CACHE = {} - _TAG_GROUPS_MTIME = mtime - return {} - - groups: Dict[str, - List[str]] = {} - if isinstance(payload, dict): - for key, value in payload.items(): - if not isinstance(key, str): - continue - name = key.strip().lower() - if not name: - continue - members: List[str] = [] - if isinstance(value, list): - for entry in value: - normalized = _normalize_tag_group_entry(entry) - if normalized: - members.append(normalized) - elif isinstance(value, str): - normalized = _normalize_tag_group_entry(value) - if normalized: - members.extend( - token.strip() for token in normalized.split(",") - if token.strip() - ) - if members: - groups[name] = members - - _TAG_GROUPS_CACHE = groups - _TAG_GROUPS_MTIME = mtime - return groups - - -def expand_tag_groups(raw_tags: Iterable[str]) -> List[str]: - """Expand tag group references (e.g., {my_group}) into member tags. - - Tag groups are defined in JSON and can be nested. Groups are referenced - with curly braces: {group_name}. - - Args: - raw_tags: Sequence of tag strings, some may reference groups like "{group_name}" - - Returns: - List of expanded tags with group references replaced - """ - groups = _load_tag_groups() - if not groups: - return [tag for tag in raw_tags if isinstance(tag, str) and tag.strip()] - - def _expand(tokens: Iterable[str], seen: Set[str]) -> List[str]: - result: List[str] = [] - for token in tokens: - if not isinstance(token, str): - continue - candidate = token.strip() - if not candidate: - continue - if candidate.startswith("{") and candidate.endswith("}") and len(candidate - ) > 2: - name = candidate[1:-1].strip().lower() - if not name: - continue - if name in seen: - log( - f"Tag group recursion detected for {{{name}}}; skipping", - file=sys.stderr - ) - continue - members = groups.get(name) - if not members: - log(f"Unknown tag group {{{name}}}", file=sys.stderr) - result.append(candidate) - continue - result.extend(_expand(members, - seen | {name})) - else: - result.append(candidate) - return result - - return _expand(raw_tags, set()) - - -def first_title_tag(source: Optional[Iterable[str]]) -> Optional[str]: - """Find the first tag starting with "title:" in a collection. - - Args: - source: Iterable of tag strings - - Returns: - First title: tag found, or None - """ - if not source: - return None - for item in source: - if not isinstance(item, str): - continue - candidate = item.strip() - if candidate and candidate.lower().startswith("title:"): - return candidate - return None - - -def apply_preferred_title(tags: List[str], preferred: Optional[str]) -> List[str]: - """Replace any title: tags with a preferred title tag. - - Args: - tags: List of tags (may contain multiple "title:" entries) - preferred: Preferred title tag to use (full "title: ..." format) - - Returns: - List with old title tags removed and preferred title added (at most once) - """ - if not preferred: - return tags - preferred_clean = preferred.strip() - if not preferred_clean: - return tags - preferred_lower = preferred_clean.lower() - filtered: List[str] = [] - has_preferred = False - for tag in tags: - candidate = tag.strip() - if not candidate: - continue - if candidate.lower().startswith("title:"): - if candidate.lower() == preferred_lower: - if not has_preferred: - filtered.append(candidate) - has_preferred = True - continue - filtered.append(candidate) - if not has_preferred: - filtered.append(preferred_clean) - return filtered - - -# ============================================================================ -# PIPEOBJECT UTILITIES (for chainable cmdlet and multi-action pipelines) -# ============================================================================ - - -def create_pipe_object_result( - source: str, - identifier: str, - file_path: str, - cmdlet_name: str, - title: Optional[str] = None, - hash_value: Optional[str] = None, - is_temp: bool = False, - parent_hash: Optional[str] = None, - tag: Optional[List[str]] = None, - **extra: Any, -) -> Dict[str, - Any]: - """Create a PipeObject-compatible result dict for pipeline chaining. - - This is a helper to emit results in the standard format that downstream - cmdlet can process (filter, tag, cleanup, etc.). - - Args: - source: Source system (e.g., 'local', 'hydrus', 'download') - identifier: Unique ID from source - file_path: Path to the file - cmdlet_name: Name of the cmdlet that created this (e.g., 'download-data', 'screen-shot') - title: Human-readable title - hash_value: SHA-256 hash of file (for integrity) - is_temp: If True, this is a temporary/intermediate artifact - parent_hash: Hash of the parent file in the chain (for provenance) - tag: List of tag values to apply - **extra: Additional fields - - Returns: - Dict with all PipeObject fields for emission - """ - store_override = extra.pop("store", None) - - result = build_file_result_payload( - title=title, - path=file_path, - hash_value=hash_value, - store=store_override if store_override is not None else source, - tag=tag, - source=source, - id=identifier, - action=f"cmdlet:{cmdlet_name}", - **extra, - ) - - if is_temp: - result["is_temp"] = True - if parent_hash: - result["parent_hash"] = parent_hash - - return result - - -def mark_as_temp(pipe_object: Dict[str, Any]) -> Dict[str, Any]: - """Mark a PipeObject dict as temporary (intermediate artifact). - - Args: - pipe_object: Result dict from cmdlet emission - - Returns: - Modified dict with is_temp=True - """ - pipe_object["is_temp"] = True - return pipe_object - - -def set_parent_hash(pipe_object: Dict[str, Any], parent_hash: str) -> Dict[str, Any]: - """Set the parent_hash for provenance tracking. - - Args: - pipe_object: Result dict - parent_hash: Parent file's hash - - Returns: - Modified dict with parent_hash set to the hash - """ - pipe_object["parent_hash"] = parent_hash - return pipe_object - - -def get_pipe_object_path(pipe_object: Any) -> Optional[str]: - """Extract file path from PipeObject, dict, or pipeline-friendly object.""" - if pipe_object is None: - return None - for attr in ("path", "target"): - if hasattr(pipe_object, attr): - value = getattr(pipe_object, attr) - if value: - return value - if isinstance(pipe_object, dict): - for key in ("path", "target"): - value = pipe_object.get(key) - if value: - return value - return None - - -def _extract_flag_value(args: Sequence[str], *flags: str) -> Optional[str]: - """Return the value for the first matching flag in args. - - This is intentionally lightweight (no cmdlet spec required) so callers in CLI/pipeline - can share the same behavior. - """ - if not args: - return None - want = {str(f).strip().lower() - for f in flags if str(f).strip()} - if not want: - return None - try: - tokens = [str(a) for a in args] - except Exception: - tokens = list(args) # type: ignore[list-item] - for i, tok in enumerate(tokens): - low = str(tok).strip().lower() - if low in want: - if i + 1 >= len(tokens): - return None - nxt = str(tokens[i + 1]) - # Allow paths like "-"? Treat missing value as None. - if not nxt.strip(): - return None - # Don't consume another flag as value. - if nxt.startswith("-"): - return None - return nxt - return None - - -def _unique_destination_path(dest: Path) -> Path: - """Generate a non-colliding destination path by appending " (N)".""" - try: - if not dest.exists(): - return dest - except Exception: - return dest - - parent = dest.parent - stem = dest.stem - suffix = dest.suffix - for i in range(1, 10_000): - candidate = parent / f"{stem} ({i}){suffix}" - try: - if not candidate.exists(): - return candidate - except Exception: - return candidate - return dest - - -def _print_live_safe_stderr(message: str) -> None: - """Print to stderr without breaking Rich Live progress output.""" - try: - from SYS.rich_display import stderr_console # type: ignore - except Exception: - return - - cm: AbstractContextManager[Any] | None = None - try: - from SYS import pipeline as _pipeline_ctx # type: ignore - - suspend = getattr(_pipeline_ctx, "suspend_live_progress", None) - candidate = suspend() if callable(suspend) else None - if isinstance(candidate, AbstractContextManager): - cm = candidate - elif candidate is not None and hasattr(candidate, "__enter__") and hasattr(candidate, "__exit__"): - cm = candidate # type: ignore[arg-type] - except Exception: - cm = None - - if cm is None: - cm = nullcontext() - - try: - console = stderr_console() - print_func = getattr(console, "print", None) - except Exception: - return - - if not callable(print_func): - return - - try: - with cm: - print_func(str(message)) - except Exception: - return - - -def apply_output_path_from_pipeobjects( - *, - cmd_name: str, - args: Sequence[str], - emits: Sequence[Any], -) -> List[Any]: - """If the user supplied `-path`, move emitted temp/PATH files there. - - This enables a dynamic pattern: - - Any cmdlet can include `SharedArgs.PATH`. - - If it emits a file-backed PipeObject (`path` exists on disk) and the item is - a temp/PATH artifact, then `-path ` will save it to that location. - - Rules: - - Only affects items whose `action` matches the current cmdlet. - - Only affects items that look like local artifacts (`is_temp` True or `store` == PATH). - - Updates the emitted object's `path` (and `target` when it points at the same file). - """ - dest_raw = _extract_flag_value(args, "-path", "--path") - if not dest_raw: - return list(emits or []) - - # Guard: users sometimes pass a URL into -path by mistake (e.g. `-path https://...`). - # Treat that as invalid for filesystem moves and avoid breaking Rich Live output. - try: - dest_str = str(dest_raw).strip() - if "://" in dest_str: - _print_live_safe_stderr( - f"Ignoring -path value that looks like a URL: {dest_str}" - ) - return list(emits or []) - except Exception: - pass - - cmd_norm = str(cmd_name or "").replace("_", "-").strip().lower() - if not cmd_norm: - return list(emits or []) - - try: - dest_hint_dir = str(dest_raw).endswith(("/", "\\")) - except Exception: - dest_hint_dir = False - - try: - dest_path = Path(str(dest_raw)).expanduser() - except Exception: - return list(emits or []) - - items = list(emits or []) - # Identify which emitted items are actually file artifacts produced by this cmdlet. - artifact_indices: List[int] = [] - artifact_paths: List[Path] = [] - for idx, item in enumerate(items): - action = str(get_field(item, "action", "") or "").strip().lower() - if not action.startswith("cmdlet:"): - continue - action_name = action.split(":", 1)[-1].strip().lower() - if action_name != cmd_norm: - continue - - store = str(get_field(item, "store", "") or "").strip().lower() - is_temp = bool(get_field(item, "is_temp", False)) - if not (is_temp or store == "path"): - continue - - src_str = get_pipe_object_path(item) - if not src_str: - continue - try: - src = Path(str(src_str)).expanduser() - except Exception: - continue - try: - if not src.exists() or not src.is_file(): - continue - except Exception: - continue - - artifact_indices.append(idx) - artifact_paths.append(src) - - if not artifact_indices: - return items - - # Decide whether the destination is a directory or a single file. - if len(artifact_indices) > 1: - # Multiple artifacts: always treat destination as a directory. - if dest_path.suffix: - dest_dir = dest_path.parent - else: - dest_dir = dest_path - try: - dest_dir.mkdir(parents=True, exist_ok=True) - except Exception as exc: - _print_live_safe_stderr( - f"Failed to create destination directory: {dest_dir} ({exc})" - ) - return items - - for idx, src in zip(artifact_indices, artifact_paths): - final = dest_dir / src.name - final = _unique_destination_path(final) - try: - if src.resolve() == final.resolve(): - _apply_saved_path_update( - items[idx], - old_path=str(src), - new_path=str(final) - ) - _print_saved_output_panel(items[idx], final) - continue - except Exception: - pass - try: - shutil.move(str(src), str(final)) - except Exception as exc: - _print_live_safe_stderr(f"Failed to save output to {final}: {exc}") - continue - _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final)) - _print_saved_output_panel(items[idx], final) - - return items - - # Single artifact: destination can be a directory or a concrete file path. - src = artifact_paths[0] - idx = artifact_indices[0] - final: Path - try: - if dest_hint_dir or (dest_path.exists() and dest_path.is_dir()): - final = dest_path / src.name - else: - final = dest_path - except Exception: - final = dest_path - - try: - final.parent.mkdir(parents=True, exist_ok=True) - except Exception as exc: - _print_live_safe_stderr( - f"Failed to create destination directory: {final.parent} ({exc})" - ) - return items - - final = _unique_destination_path(final) - try: - if src.resolve() != final.resolve(): - shutil.move(str(src), str(final)) - except Exception as exc: - _print_live_safe_stderr(f"Failed to save output to {final}: {exc}") - return items - - _apply_saved_path_update(items[idx], old_path=str(src), new_path=str(final)) - _print_saved_output_panel(items[idx], final) - return items - - -def _print_saved_output_panel(item: Any, final_path: Path) -> None: - """When -path is used, print a Rich panel summarizing the saved output. - - Shows: Title, Location, Hash. - Best-effort: reads existing fields first to avoid recomputing hashes. - """ - try: - from rich.panel import Panel # type: ignore - from rich.table import Table # type: ignore - from SYS.rich_display import stderr_console # type: ignore - except Exception: - return - - # If Rich Live progress is active, pause it while printing so the panel - # doesn't get overwritten/truncated by Live's cursor control. - cm: AbstractContextManager[Any] | None = None - try: - from SYS import pipeline as _pipeline_ctx # type: ignore - - suspend = getattr(_pipeline_ctx, "suspend_live_progress", None) - cm_candidate = suspend() if callable(suspend) else None - if isinstance(cm_candidate, AbstractContextManager): - cm = cm_candidate - elif cm_candidate is not None and hasattr(cm_candidate, "__enter__") and hasattr(cm_candidate, "__exit__"): - cm = cm_candidate # type: ignore[arg-type] - except Exception: - cm = None - - if cm is None: - cm = nullcontext() - - try: - location = str(final_path) - except Exception: - location = "" - - title = "" - try: - title = str(get_field(item, "title") or get_field(item, "name") or "").strip() - except Exception: - title = "" - if not title: - try: - title = str(final_path.stem or final_path.name) - except Exception: - title = "" - - file_hash = "" - try: - file_hash = str(get_field(item, - "hash") or get_field(item, - "sha256") or "").strip() - except Exception: - file_hash = "" - if not file_hash: - try: - from SYS.utils import sha256_file # type: ignore - - file_hash = str(sha256_file(final_path) or "").strip() - except Exception: - file_hash = "" - - grid = Table.grid(padding=(0, 1)) - grid.add_column(justify="right", style="bold") - grid.add_column() - grid.add_row("Title", title or "(unknown)") - grid.add_row("Location", location or "(unknown)") - grid.add_row("Hash", file_hash or "(unknown)") - - try: - console = stderr_console() - print_func = getattr(console, "print", None) - except Exception: - return - - if not callable(print_func): - return - - try: - with cm: - print_func(Panel(grid, title="Saved", expand=False)) - except Exception: - return - - -def _apply_saved_path_update(item: Any, *, old_path: str, new_path: str) -> None: - """Update a PipeObject-like item after its backing file has moved.""" - old_str = str(old_path) - new_str = str(new_path) - if isinstance(item, dict): - try: - if str(item.get("path") or "") == old_str: - item["path"] = new_str - except Exception: - pass - try: - if str(item.get("target") or "") == old_str: - item["target"] = new_str - except Exception: - pass - try: - extra = item.get("extra") - if isinstance(extra, dict): - if str(extra.get("target") or "") == old_str: - extra["target"] = new_str - if str(extra.get("path") or "") == old_str: - extra["path"] = new_str - except Exception: - pass - return - - # models.PipeObject or PipeObject-ish - try: - if getattr(item, "path", None) == old_str: - setattr(item, "path", new_str) - except Exception: - pass - try: - extra = getattr(item, "extra", None) - if isinstance(extra, dict): - if str(extra.get("target") or "") == old_str: - extra["target"] = new_str - if str(extra.get("path") or "") == old_str: - extra["path"] = new_str - except Exception: - pass - - -def get_pipe_object_hash(pipe_object: Any) -> Optional[str]: - """Extract file hash from PipeObject, dict, or pipeline-friendly object.""" - if pipe_object is None: - return None - for attr in ("hash", - ): - if hasattr(pipe_object, attr): - value = getattr(pipe_object, attr) - if value: - return value - if isinstance(pipe_object, dict): - for key in ("hash", - ): - value = pipe_object.get(key) - if value: - return value - return None - - -def normalize_result_input(result: Any) -> List[Dict[str, Any]]: - """Normalize input result to a list of dicts. - - Handles: - - None -> [] - - Dict -> [dict] - - List of dicts -> list as-is - - PipeObject -> [dict] - - List of PipeObjects -> list of dicts - - Args: - result: Result from piped input - - Returns: - List of result dicts (may be empty) - """ - if result is None: - return [] - - # Single dict - if isinstance(result, dict): - return [result] - - # List - convert each item to dict if needed - if isinstance(result, list): - output = [] - for item in result: - if isinstance(item, dict): - output.append(item) - elif hasattr(item, "to_dict"): - output.append(item.to_dict()) - else: - # Try as-is - output.append(item) - return output - - # PipeObject or other object with to_dict - if hasattr(result, "to_dict"): - return [result.to_dict()] - - # Fallback: wrap it - if isinstance(result, dict): - return [result] - - return [] - - -def normalize_result_items( - result: Any, - *, - include_falsey_single: bool = False, -) -> List[Any]: - """Normalize piped input to a raw item list without converting item types.""" - if isinstance(result, list): - return list(result) - if result is None: - return [] - if include_falsey_single or result: - return [result] - return [] - - -def value_has_content(value: Any) -> bool: - """Return True when a value should be treated as present for payload building.""" - if value is None: - return False - if isinstance(value, str): - return bool(value.strip()) - if isinstance(value, (list, tuple, set)): - return len(value) > 0 - return True - - -def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]: - """Filter results by temporary status. - - Args: - results: List of result dicts or PipeObjects - include_temp: If True, keep temp files; if False, exclude them - - Returns: - Filtered list - """ - if include_temp: - return results - - filtered = [] - for result in results: - is_temp = False - - # Check PipeObject - if hasattr(result, "is_temp"): - is_temp = result.is_temp - # Check dict - elif isinstance(result, dict): - is_temp = result.get("is_temp", False) - - if not is_temp: - filtered.append(result) - - return filtered - - -def merge_sequences(*sources: Optional[Iterable[Any]], - case_sensitive: bool = True) -> list[str]: - """Merge iterable sources while preserving order and removing duplicates.""" - seen: set[str] = set() - merged: list[str] = [] - for source in sources: - if not source: - continue - if isinstance(source, str) or not isinstance(source, IterableABC): - iterable = [source] - else: - iterable = source - for value in iterable: - if value is None: - continue - text = str(value).strip() - if not text: - continue - key = text if case_sensitive else text.lower() - if key in seen: - continue - seen.add(key) - merged.append(text) - return merged - - -def collapse_namespace_tags( - tags: Optional[Iterable[Any]], - namespace: str, - prefer: str = "last" -) -> list[str]: - """Reduce tags so only one entry for a given namespace remains. - - Keeps either the first or last occurrence (default last) while preserving overall order - for non-matching tags. Useful for ensuring a single title: tag. - """ - if not tags: - return [] - ns = str(namespace or "").strip().lower() - if not ns: - return list(tags) if isinstance(tags, list) else list(tags) - - prefer_last = str(prefer or "last").lower() != "first" - ns_prefix = ns + ":" - - items = list(tags) - if prefer_last: - kept: list[str] = [] - seen_ns = False - for tag in reversed(items): - text = str(tag) - if text.lower().startswith(ns_prefix): - if seen_ns: - continue - seen_ns = True - kept.append(text) - kept.reverse() - return kept - else: - kept_ns = False - result: list[str] = [] - for tag in items: - text = str(tag) - if text.lower().startswith(ns_prefix): - if kept_ns: - continue - kept_ns = True - result.append(text) - return result -def extract_tag_from_result(result: Any) -> list[str]: - """Extract all tags from a result dict or PipeObject. - - Handles mixed types (lists, sets, strings) and various field names. - """ - tag: list[str] = [] - - def _extend(candidate: Any) -> None: - if not candidate: - return - if isinstance(candidate, (list, set, tuple)): - tag.extend(str(t) for t in candidate if t is not None) - elif isinstance(candidate, str): - tag.append(candidate) - - if isinstance(result, models.PipeObject): - tag.extend(result.tag or []) - if isinstance(result.extra, dict): - _extend(result.extra.get("tag")) - if isinstance(result.metadata, dict): - _extend(result.metadata.get("tag")) - _extend(result.metadata.get("tags")) - elif hasattr(result, "tag"): - # Handle objects with tag attribute (e.g. SearchResult) - _extend(getattr(result, "tag")) - - if isinstance(result, dict): - _extend(result.get("tag")) - _extend(result.get("tags")) - - extra = result.get("extra") - if isinstance(extra, dict): - _extend(extra.get("tag")) - _extend(extra.get("tags")) - - fm = result.get("full_metadata") or result.get("metadata") - if isinstance(fm, dict): - _extend(fm.get("tag")) - _extend(fm.get("tags")) - - return merge_sequences(tag, case_sensitive=True) - - -def extract_title_from_result(result: Any) -> Optional[str]: - """Extract the title from a result dict or PipeObject.""" - if isinstance(result, models.PipeObject): - return result.title - elif hasattr(result, "title"): - return getattr(result, "title") - elif isinstance(result, dict): - return result.get("title") - return None - - -def extract_url_from_result(result: Any) -> list[str]: - """Extract all unique URLs from a result dict or PipeObject. - - Handles mixed types (lists, strings) and various field names (url, source_url, webpage_url). - Centralizes extraction logic for cmdlets like download-file, add-file, get-url. - """ - url: list[str] = [] - - def _extend(candidate: Any) -> None: - if not candidate: - return - if isinstance(candidate, list): - url.extend(candidate) - elif isinstance(candidate, str): - url.append(candidate) - - # Priority 1: PipeObject (structured data) - if isinstance(result, models.PipeObject): - _extend(result.url) - _extend(result.source_url) - # Also check extra and metadata for legacy or rich captures - if isinstance(result.extra, dict): - _extend(result.extra.get("url")) - _extend(result.extra.get("source_url")) - if isinstance(result.metadata, dict): - _extend(result.metadata.get("url")) - _extend(result.metadata.get("source_url")) - _extend(result.metadata.get("webpage_url")) - if isinstance(getattr(result, "full_metadata", None), dict): - fm = getattr(result, "full_metadata", None) - if isinstance(fm, dict): - _extend(fm.get("url")) - _extend(fm.get("source_url")) - _extend(fm.get("webpage_url")) - - # Priority 2: Generic objects with .url or .source_url attribute - elif hasattr(result, "url") or hasattr(result, "source_url"): - _extend(getattr(result, "url", None)) - _extend(getattr(result, "source_url", None)) - - # Priority 3: Dictionary - if isinstance(result, dict): - _extend(result.get("url")) - _extend(result.get("source_url")) - _extend(result.get("webpage_url")) - - extra = result.get("extra") - if isinstance(extra, dict): - _extend(extra.get("url")) - - fm = result.get("full_metadata") or result.get("metadata") - if isinstance(fm, dict): - _extend(fm.get("url")) - _extend(fm.get("source_url")) - _extend(fm.get("webpage_url")) - - from SYS.metadata import normalize_urls - return normalize_urls(url) - - -def merge_urls(existing: Any, incoming: Sequence[Any]) -> list[str]: - """Merge URL values into a normalized, de-duplicated list.""" - from SYS.metadata import normalize_urls - - merged: list[str] = [] - for value in normalize_urls(existing): - if value not in merged: - merged.append(value) - for value in normalize_urls(list(incoming or [])): - if value not in merged: - merged.append(value) - return merged - - -def remove_urls(existing: Any, remove: Sequence[Any]) -> list[str]: - """Remove URL values from an existing URL field and return survivors.""" - from SYS.metadata import normalize_urls - - current = normalize_urls(existing) - remove_set = {value for value in normalize_urls(list(remove or [])) if value} - if not remove_set: - return current - return [value for value in current if value not in remove_set] - - -def set_item_urls(item: Any, urls: Sequence[Any]) -> None: - """Persist normalized URL values back onto a dict/object result item.""" - normalized = merge_urls([], list(urls or [])) - payload: Any = normalized[0] if len(normalized) == 1 else list(normalized) - - try: - if isinstance(item, dict): - item["url"] = payload - return - if hasattr(item, "url"): - setattr(item, "url", payload) - except Exception: - return - - -def extract_relationships(result: Any) -> Optional[Dict[str, Any]]: - if isinstance(result, models.PipeObject): - relationships = result.get_relationships() - return relationships or None - if isinstance(result, dict): - relationships = result.get("relationships") - if isinstance(relationships, dict) and relationships: - return relationships - return None - - -def extract_duration(result: Any) -> Optional[float]: - duration = None - if isinstance(result, models.PipeObject): - duration = result.duration - elif isinstance(result, dict): - duration = result.get("duration") - if duration is None: - metadata = result.get("metadata") - if isinstance(metadata, dict): - duration = metadata.get("duration") - if duration is None: - return None - try: - return float(duration) - except (TypeError, ValueError): - return None - - -def coerce_to_pipe_object( - value: Any, - default_path: Optional[str] = None -) -> models.PipeObject: - """Normalize any incoming result to a PipeObject for single-source-of-truth state. - - Uses hash+store canonical pattern. - """ - if isinstance(value, models.PipeObject): - return value - - known_keys = { - "hash", - "store", - "tag", - "title", - "url", - "source_url", - "duration", - "metadata", - "warnings", - "path", - "relationships", - "is_temp", - "action", - "parent_hash", - } - - # Convert common object-like results into a dict so we can preserve fields like - # hash/store/url when they come from result tables (e.g., get-url emits UrlItem). - # - # Priority: - # 1) explicit to_dict() - # 2) best-effort attribute extraction for known PipeObject-ish fields - if hasattr(value, "to_dict"): - value = value.to_dict() - elif not isinstance(value, dict): - try: - obj_map: Dict[str, - Any] = {} - for k in ( - "hash", - "store", - "provider", - "prov", - "tag", - "title", - "url", - "source_url", - "duration", - "duration_seconds", - "metadata", - "full_metadata", - "warnings", - "path", - "target", - "relationships", - "is_temp", - "action", - "parent_hash", - "extra", - "media_kind", - ): - if hasattr(value, k): - obj_map[k] = getattr(value, k) - if obj_map: - value = obj_map - except Exception: - pass - - if isinstance(value, dict): - # Extract hash and store (canonical identifiers) - hash_val = value.get("hash") - store_val = value.get("store") or "PATH" - if not store_val or store_val == "PATH": - try: - extra_store = value.get("extra", - {}).get("store") - except Exception: - extra_store = None - if extra_store: - store_val = extra_store - - # If no hash, try to compute from path or use placeholder - if not hash_val: - path_val = value.get("path") - if path_val: - try: - from SYS.utils import sha256_file - from pathlib import Path - - hash_val = sha256_file(Path(path_val)) - except Exception: - hash_val = "unknown" - else: - hash_val = "unknown" - - # Extract title from filename if not provided - title_val = value.get("title") - if not title_val: - path_val = value.get("path") - if path_val: - try: - from pathlib import Path - - title_val = Path(path_val).stem - except Exception: - pass - - extra = { - k: v - for k, v in value.items() if k not in known_keys - } - - # Extract URL: prefer direct url field, then url list - from SYS.metadata import normalize_urls - - url_list = normalize_urls(value.get("url")) - url_val = url_list[0] if url_list else None - if len(url_list) > 1: - extra["url"] = url_list - - # Extract relationships - rels = value.get("relationships") or {} - - # Canonical tag: accept list or single string - tag_val: list[str] = [] - if "tag" in value: - raw_tag = value["tag"] - if isinstance(raw_tag, list): - tag_val = [str(t) for t in raw_tag if t is not None] - elif isinstance(raw_tag, str): - tag_val = [raw_tag] - - # Consolidate path: prefer explicit path key, but NOT target if it's a URL - path_val = value.get("path") - # Only use target as path if it's not a URL (url should stay in url field) - if not path_val and "target" in value: - target = value["target"] - if target and not (isinstance(target, - str) and (target.startswith("http://") - or target.startswith("https://"))): - path_val = target - - # If the path value is actually a URL, move it to url_val and clear path_val - try: - if isinstance(path_val, - str) and (path_val.startswith("http://") - or path_val.startswith("https://")): - # Prefer existing url_val if present, otherwise move path_val into url_val - if not url_val: - url_val = path_val - path_val = None - except Exception: - pass - - # Extract media_kind if available - if "media_kind" in value: - extra["media_kind"] = value["media_kind"] - - pipe_obj = models.PipeObject( - hash=hash_val, - store=store_val, - plugin=str( - value.get("plugin") - or value.get("prov") - or value.get("source") - or extra.get("plugin") - or extra.get("source") - or "" - ).strip() or None, - tag=tag_val, - title=title_val, - url=url_val, - source_url=value.get("source_url"), - duration=value.get("duration") or value.get("duration_seconds"), - metadata=value.get("metadata") or value.get("full_metadata") or {}, - warnings=list(value.get("warnings") or []), - path=path_val, - relationships=rels, - is_temp=bool(value.get("is_temp", - False)), - action=value.get("action"), - parent_hash=value.get("parent_hash"), - extra=extra, - ) - - return pipe_obj - - # Fallback: build from path argument or bare value - hash_val = "unknown" - path_val = default_path or getattr(value, "path", None) - url_val: Optional[str] = None - title_val = None - - # If the raw value is a string, treat it as either a URL or a file path. - # This is important for @-selection results that are plain URL strings. - if isinstance(value, str): - s = value.strip() - if s.lower().startswith(("http://", "https://")): - url_val = s - path_val = None - else: - path_val = s - - if path_val and path_val != "unknown": - try: - from SYS.utils import sha256_file - from pathlib import Path - - path_obj = Path(path_val) - hash_val = sha256_file(path_obj) - # Extract title from filename (without extension) - title_val = path_obj.stem - except Exception: - pass - - # When coming from a raw URL string, mark it explicitly as URL. - # Otherwise treat it as a local path. - store_val = "URL" if url_val else "PATH" - - pipe_obj = models.PipeObject( - hash=hash_val, - store=store_val, - plugin=None, - path=str(path_val) if path_val and path_val != "unknown" else None, - title=title_val, - url=url_val, - source_url=url_val, - tag=[], - extra={}, - ) - - return pipe_obj - - -def propagate_metadata( - previous_items: Sequence[Any], - new_items: Sequence[Any] -) -> List[Any]: - """Merge metadata/tags from previous pipeline stage into new items. - - Implements "sticky metadata": items generated by a transformation (download, convert) - should inherit rich info (lyrics, art, tags) from their source. - - Strategies: - A. Hash Match: If inputs/outputs share a hash, they are the same item. - B. Index Match: If lists are same length, assume 1:1 mapping (heuristic). - C. Explicit Parent: If output has `parent_hash`, link to input with that hash. - """ - if not previous_items or not new_items: - return list(new_items) - - try: - prev_normalized = [coerce_to_pipe_object(p) for p in previous_items] - except Exception: - return list(new_items) - - prev_by_hash: Dict[str, models.PipeObject] = {} - for p_obj in prev_normalized: - if p_obj.hash and p_obj.hash != "unknown": - prev_by_hash[p_obj.hash] = p_obj - - normalized: List[Any] = [] - - # Pre-calculate length matching for heuristic - is_same_length = len(new_items) == len(prev_normalized) - - for i, item in enumerate(new_items): - if isinstance(item, dict) and item.get("_skip_metadata_propagation"): - normalized.append(item) - continue - try: - obj = coerce_to_pipe_object(item) - except Exception: - normalized.append(item) # Should not happen given coerce guards - continue - - parent: Optional[models.PipeObject] = None - - # Strategy A: Precise Hash Match - if obj.hash in prev_by_hash: - parent = prev_by_hash[obj.hash] - - # Strategy B: Index Match (Heuristic) - if not parent and is_same_length: - parent = prev_normalized[i] - - # Strategy C: Explicit Parent Hash - if not parent and obj.parent_hash and obj.parent_hash in prev_by_hash: - parent = prev_by_hash[obj.parent_hash] - - if parent: - # 1. Tags: Merge unique tags - if parent.tag: - if not obj.tag: - obj.tag = list(parent.tag) - else: - curr_tags = {str(t).lower() for t in obj.tag} - for pt in parent.tag: - if str(pt).lower() not in curr_tags: - obj.tag.append(pt) - - # 2. Metadata: Merge missing keys - if parent.metadata: - if not obj.metadata: - obj.metadata = parent.metadata.copy() - else: - for mk, mv in parent.metadata.items(): - if mk not in obj.metadata: - obj.metadata[mk] = mv - - # 3. Source URL: Propagate if missing - if parent.source_url and not obj.source_url: - obj.source_url = parent.source_url - elif parent.url and not obj.source_url and not obj.url: - # If parent had a URL and child has none, it's likely the source - obj.source_url = parent.url - - # 4. Relationships: Merge missing keys - if parent.relationships: - if not obj.relationships: - obj.relationships = parent.relationships.copy() - else: - for rk, rv in parent.relationships.items(): - if rk not in obj.relationships: - obj.relationships[rk] = rv - - # 5. Extra (Notes/etc): Merge missing keys - # Important for passing 'notes' payload (lyrics, captions) - if parent.extra: - if not obj.extra: - obj.extra = parent.extra.copy() - else: - # Recursive merge for 'notes' dict specifically? - # For now just shallow merge keys, but handle 'notes' specially if valid. - for ek, ev in parent.extra.items(): - if ek not in obj.extra: - obj.extra[ek] = ev - elif ek == "notes" and isinstance(ev, dict) and isinstance(obj.extra[ek], dict): - # Merge notes dict - for nk, nv in ev.items(): - if nk not in obj.extra[ek]: - obj.extra[ek][nk] = nv - - normalized.append(obj) - - return normalized - - -def register_url_with_local_library( - pipe_obj: models.PipeObject, - config: Dict[str, - Any] -) -> bool: - """Register url with a file in the local library database. - - This is called automatically by download cmdlet to ensure url are persisted - without requiring a separate add-url step in the pipeline. - - Args: - pipe_obj: PipeObject with path and url - config: Config dict containing local library path - - Returns: - True if url were registered, False otherwise - """ - # Folder store removed; local library URL registration is disabled. - return False -def check_url_exists_in_storage( - urls: Sequence[str], - storage: Any, - hydrus_available: bool, - final_output_dir: Optional[Path] = None, - *, - auto_continue_duplicates: bool = True, - force_prompt_in_pipeline: bool = False, -) -> bool: - """Pre-flight check to see if URLs already exist in storage. - - Args: - urls: List of URLs to check - storage: The storage interface - hydrus_available: Whether Hydrus is available - final_output_dir: Final output directory (to skip if same as storage) - - Returns: - True if check passed (user said yes or no dups), False if user said no (stop). - """ - if storage is None: - debug("Bulk URL preflight skipped: storage unavailable") - return True - - try: - current_cmd_text = pipeline_context.get_current_command_text("") - except Exception: - current_cmd_text = "" - - try: - stage_ctx = pipeline_context.get_stage_context() - except Exception: - stage_ctx = None - - in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or ""))) - start_time = time.monotonic() - time_budget = 45.0 - if in_pipeline: - try: - already_checked = bool( - pipeline_context.load_value( - "preflight.url_duplicates.checked", default=False - ) - ) - except Exception: - already_checked = False - - if already_checked: - debug("Bulk URL preflight: already checked in pipeline; skipping duplicate check") - return True - - def _load_preflight_cache() -> Dict[str, Any]: - try: - existing = pipeline_context.load_value("preflight", default=None) - except Exception: - existing = None - return existing if isinstance(existing, dict) else {} - - def _store_preflight_cache(cache: Dict[str, Any]) -> None: - try: - pipeline_context.store_value("preflight", cache) - except Exception: - pass - - def _mark_preflight_checked() -> None: - if not in_pipeline: - return - try: - pipeline_context.store_value("preflight.url_duplicates.checked", True) - except Exception: - pass - preflight_cache = _load_preflight_cache() - preflight_cache["url_duplicates_checked"] = True - url_dup_cache = preflight_cache.get("url_duplicates") - if not isinstance(url_dup_cache, dict): - url_dup_cache = {} - url_dup_cache["checked"] = True - preflight_cache["url_duplicates"] = url_dup_cache - _store_preflight_cache(preflight_cache) - - def _timed_out(reason: str) -> bool: - try: - if (time.monotonic() - start_time) >= time_budget: - debug( - f"Bulk URL preflight timed out after {time_budget:.0f}s ({reason}); continuing" - ) - _mark_preflight_checked() - return True - except Exception: - return False - return False - - if in_pipeline and auto_continue_duplicates: - try: - cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="") - cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None) - except Exception: - cached_cmd = "" - cached_decision = None - - if (not force_prompt_in_pipeline) and cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""): - _mark_preflight_checked() - if bool(cached_decision): - return True - try: - pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0) - except Exception: - pass - return False - - unique_urls: List[str] = [] - for u in urls or []: - s = str(u or "").strip() - if s and s not in unique_urls: - unique_urls.append(s) - if len(unique_urls) == 0: - return True - - try: - from SYS.metadata import normalize_urls - except Exception: - normalize_urls = None # type: ignore[assignment] - - def _httpish(value: str) -> bool: - try: - return bool(value) and (value.startswith("http://") or value.startswith("https://")) - except Exception: - return False - - def _normalize_url_for_search(value: str) -> str: - url = str(value or "").strip() - - # Strip fragment (e.g., #t=10) before matching - url = url.split("#", 1)[0] - - # Strip common time/tracking query params for matching - try: - parsed = urlparse(url) - except Exception: - parsed = None - - if parsed is not None and parsed.query: - time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"} - tracking_prefixes = ("utm_",) - try: - pairs = parse_qsl(parsed.query, keep_blank_values=True) - filtered = [] - for key, val in pairs: - key_norm = str(key or "").lower() - if key_norm in time_keys: - continue - if key_norm.startswith(tracking_prefixes): - continue - filtered.append((key, val)) - if filtered: - url = urlunparse(parsed._replace(query=urlencode(filtered, doseq=True))) - else: - url = urlunparse(parsed._replace(query="")) - except Exception: - pass - - # Remove protocol (http://, https://, ftp://, etc.) - url = re.sub(r"^[a-z][a-z0-9+.-]*://", "", url, flags=re.IGNORECASE) - - # Remove www. prefix (case-insensitive) - url = re.sub(r"^www\.", "", url, flags=re.IGNORECASE) - - return url.lower() - - def _expand_url_variants(value: str) -> List[str]: - if not _httpish(value): - return [] - - try: - parsed = urlparse(value) - except Exception: - return [] - - if parsed.scheme.lower() not in {"http", "https"}: - return [] - - out: List[str] = [] - - def _add_variant(candidate: str) -> None: - _maybe_add(candidate) - try: - lower = str(candidate or "").lower() - except Exception: - lower = "" - if lower and lower != candidate: - _maybe_add(lower) - - try: - parsed_candidate = urlparse(candidate) - except Exception: - parsed_candidate = None - - if parsed_candidate is None: - return - - host = (parsed_candidate.hostname or "").strip().lower() - if host.startswith("www."): - host = host[4:] - if host: - netloc = host - try: - if parsed_candidate.port: - netloc = f"{netloc}:{parsed_candidate.port}" - except Exception: - pass - try: - if parsed_candidate.username or parsed_candidate.password: - userinfo = parsed_candidate.username or "" - if parsed_candidate.password: - userinfo = f"{userinfo}:{parsed_candidate.password}" - if userinfo: - netloc = f"{userinfo}@{netloc}" - except Exception: - pass - alt = urlunparse(parsed_candidate._replace(netloc=netloc)) - _maybe_add(alt) - try: - lower_alt = alt.lower() - except Exception: - lower_alt = "" - if lower_alt and lower_alt != alt: - _maybe_add(lower_alt) - - def _maybe_add(candidate: str) -> None: - if not candidate or candidate == value: - return - if candidate not in out: - out.append(candidate) - - if parsed.fragment: - _add_variant(urlunparse(parsed._replace(fragment=""))) - - time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"} - tracking_prefixes = ("utm_",) - - try: - query_pairs = parse_qsl(parsed.query, keep_blank_values=True) - except Exception: - query_pairs = [] - - if query_pairs or parsed.fragment: - filtered_pairs = [] - removed = False - for key, val in query_pairs: - key_norm = str(key or "").lower() - if key_norm in time_keys: - removed = True - continue - if key_norm.startswith(tracking_prefixes): - removed = True - continue - filtered_pairs.append((key, val)) - - if removed: - new_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else "" - _add_variant(urlunparse(parsed._replace(query=new_query, fragment=""))) - - return out - - def _dedupe_needles(raw_needles: Sequence[str]) -> List[str]: - output: List[str] = [] - seen: set[str] = set() - for candidate in (raw_needles or []): - candidate_text = str(candidate or "").strip() - if not candidate_text: - continue - key = candidate_text.lower() - if key in seen: - continue - seen.add(key) - output.append(candidate_text) - return output - - url_needles: Dict[str, List[str]] = {} - for u in unique_urls: - needles: List[str] = [] - if normalize_urls is not None: - try: - needles.extend([n for n in (normalize_urls(u) or []) if isinstance(n, str)]) - except Exception: - needles = [] - if not needles: - needles = [u] - filtered: List[str] = [] - for n in needles: - n2 = str(n or "").strip() - if not n2: - continue - if not _httpish(n2): - continue - if n2 not in filtered: - filtered.append(n2) - lowered: List[str] = [] - for n2 in filtered: - try: - lower = n2.lower() - except Exception: - lower = "" - if lower and lower != n2 and lower not in filtered and lower not in lowered: - lowered.append(lower) - normalized: List[str] = [] - for n2 in filtered: - norm = _normalize_url_for_search(n2) - if norm and norm not in normalized and norm not in filtered: - normalized.append(norm) - expanded: List[str] = [] - for n2 in filtered: - for extra in _expand_url_variants(n2): - if extra not in expanded and extra not in filtered and extra not in lowered: - expanded.append(extra) - norm_extra = _normalize_url_for_search(extra) - if ( - norm_extra - and norm_extra not in normalized - and norm_extra not in filtered - and norm_extra not in expanded - and norm_extra not in lowered - ): - normalized.append(norm_extra) - - combined = filtered + expanded + lowered + normalized - deduped = _dedupe_needles(combined) - url_needles[u] = deduped if deduped else [u] - - if in_pipeline: - preflight_cache = _load_preflight_cache() - url_dup_cache = preflight_cache.get("url_duplicates") - if not isinstance(url_dup_cache, dict): - url_dup_cache = {} - cached_urls = url_dup_cache.get("urls") - cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() - - if cached_set: - all_cached = True - for original_url, needles in url_needles.items(): - original_cached = str(original_url or "") in cached_set - needles_cached = True - if original_cached: - for needle in (needles or []): - needle_text = str(needle or "") - if not needle_text: - continue - if needle_text not in cached_set: - needles_cached = False - break - else: - needles_cached = False - - if original_cached and needles_cached: - continue - - all_cached = False - break - - if all_cached: - debug("Bulk URL preflight: cached for pipeline; skipping duplicate check") - _mark_preflight_checked() - return True - - if _timed_out("before backend scan"): - return True - - # Use bulk mode only if we have a significant number of URLs. - # For small sets (1-3 URLs), individual targeted searches are faster - # and more accurate than scanning all files with URLs in the backend. - bulk_mode = len(unique_urls) > 3 - - def _build_bulk_patterns(needles_map: Dict[str, List[str]], max_per_url: int = 3, max_total: int = 240) -> List[str]: - patterns: List[str] = [] - for _original, needles in needles_map.items(): - for needle in (needles or [])[:max_per_url]: - needle_text = str(needle or "").strip() - if not needle_text: - continue - if needle_text not in patterns: - patterns.append(needle_text) - if len(patterns) >= max_total: - return patterns - return patterns - - bulk_patterns = _build_bulk_patterns(url_needles) - - def _match_normalized_url(pattern_text: str, candidate_url: str) -> bool: - pattern_norm = _normalize_url_for_search(pattern_text) - candidate_norm = _normalize_url_for_search(candidate_url) - if not pattern_norm or not candidate_norm: - return False - if pattern_norm == candidate_norm: - return True - return pattern_norm in candidate_norm - - def _extract_urls_from_hit( - hit: Any, - backend: Any, - *, - allow_backend_lookup: bool = True, - ) -> List[str]: - url_values: List[str] = [] - try: - raw_urls = get_field(hit, "known_urls") or get_field(hit, "urls") or get_field(hit, "url") - if isinstance(raw_urls, str) and raw_urls.strip(): - url_values.append(raw_urls.strip()) - elif isinstance(raw_urls, (list, tuple, set)): - for item in raw_urls: - if isinstance(item, str) and item.strip(): - url_values.append(item.strip()) - except Exception: - url_values = [] - - if url_values or not allow_backend_lookup: - return url_values - - try: - file_hash = get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or "" - except Exception: - file_hash = "" - - if file_hash: - try: - fetched = backend.get_url(str(file_hash)) - if isinstance(fetched, str) and fetched.strip(): - url_values.append(fetched.strip()) - elif isinstance(fetched, (list, tuple, set)): - for item in fetched: - if isinstance(item, str) and item.strip(): - url_values.append(item.strip()) - except Exception: - pass - - return url_values - - def _build_display_row_for_hit( - hit: Any, - backend_name: str, - original_url: str, - ) -> Dict[str, Any]: - try: - from SYS.result_table import build_display_row - extracted = build_display_row(hit, keys=["title", "store", "hash", "ext", "size"]) - except Exception: - extracted = {} - - try: - title = extracted.get("title") or get_field(hit, "title") or get_field(hit, "name") or get_field(hit, "target") or get_field(hit, "path") or "(exists)" - except Exception: - title = "(exists)" - - try: - file_hash = extracted.get("hash") or get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or "" - except Exception: - file_hash = "" - - ext = extracted.get("ext") if isinstance(extracted, dict) else "" - size_val = extracted.get("size") if isinstance(extracted, dict) else None - - return build_table_result_payload( - title=str(title), - columns=[ - ("Title", str(title)), - ("Store", str(get_field(hit, "store") or backend_name)), - ("Hash", str(file_hash or "")), - ("Ext", str(ext or "")), - ("Size", size_val), - ("URL", original_url), - ], - store=str(get_field(hit, "store") or backend_name), - hash=str(file_hash or ""), - ext=str(ext or ""), - size=size_val, - url=original_url, - ) - - def _search_backend_url_hits( - backend: Any, - backend_name: str, - original_url: str, - needles: Sequence[str], - ) -> Optional[Dict[str, Any]]: - backend_hits: List[Dict[str, Any]] = [] - - # 1) Try exact match first (no wildcards). - # This is extremely fast for Hydrus and others that support direct URL lookup. - for needle in (needles or [])[:5]: - needle_stripped = str(needle or "").strip() - if not needle_stripped or not _httpish(needle_stripped): - continue - try: - # Use 'url:' prefix to ensure storage layers (like Hydrus) recognize it as a URL lookup - query = f"url:{needle_stripped}" - backend_hits = backend.search(query, limit=1, minimal=True) or [] - if backend_hits: - return _build_display_row_for_hit(backend_hits[0], backend_name, original_url) - except Exception: - continue - - # 2) Fallback to wildcard substring search for normalized variants. - # This is for backends where the URL might be stored differently (partial match). - for needle in (needles or [])[:3]: - needle_text = str(needle or "").strip() - if not needle_text: - continue - search_needle = _normalize_url_for_search(needle_text) or needle_text - query = f"url:*{search_needle}*" - try: - backend_hits = backend.search(query, limit=1, minimal=True) or [] - if backend_hits: - break - except Exception: - continue - - if not backend_hits: - return None - - hit = backend_hits[0] - return _build_display_row_for_hit(hit, backend_name, original_url) - - backend_names: List[str] = [] - try: - backend_names_all = storage.list_searchable_backends() - except Exception: - backend_names_all = [] - - for backend_name in backend_names_all: - try: - backend = storage[backend_name] - except Exception: - continue - - try: - if str(backend_name).strip().lower() == "temp": - continue - except Exception: - pass - - try: - backend_location = getattr(backend, "_location", None) - if backend_location and final_output_dir: - backend_path = Path(str(backend_location)).expanduser().resolve() - temp_path = Path(str(final_output_dir)).expanduser().resolve() - if backend_path == temp_path: - continue - except Exception: - pass - - backend_names.append(backend_name) - - if not backend_names: - debug("Bulk URL preflight skipped: no searchable backends") - return True - - try: - debug_panel( - "URL preflight", - [ - ("url_count", len(unique_urls)), - ("pipeline", in_pipeline), - ("bulk_mode", bulk_mode), - ("backends", ", ".join(str(name) for name in backend_names)), - ], - border_style="yellow", - ) - except Exception: - pass - - seen_pairs: set[tuple[str, str]] = set() - matched_urls: set[str] = set() - match_rows: List[Dict[str, Any]] = [] - max_rows = 200 - - hydrus_provider = None - try: - from PluginCore.registry import get_plugin - - hydrus_provider = get_plugin("hydrusnetwork", config) - except Exception: - hydrus_provider = None - - for backend_name in backend_names: - if _timed_out("backend scan"): - return True - if len(match_rows) >= max_rows: - break - try: - backend = storage[backend_name] - except Exception: - continue - - is_hydrus_backend = False - try: - is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name))) - except Exception: - is_hydrus_backend = False - if not is_hydrus_backend: - try: - is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork" - except Exception: - is_hydrus_backend = False - - if is_hydrus_backend: - if not hydrus_available: - debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup") - - if _timed_out("hydrus scan"): - return True - - for original_url, needles in url_needles.items(): - if _timed_out("hydrus per-url scan"): - return True - if len(match_rows) >= max_rows: - break - if (original_url, str(backend_name)) in seen_pairs: - continue - - found_hash: Optional[str] = None - found = False - lookup_exact = getattr(backend, "find_hashes_by_url", None) - if callable(lookup_exact): - for needle in [original_url, *(needles or [])][:7]: - needle_text = str(needle or "").strip() - if not _httpish(needle_text): - continue - try: - exact_hashes = lookup_exact(needle_text) or [] - except Exception: - continue - if not isinstance(exact_hashes, list) or not exact_hashes: - continue - try: - found_hash = str(exact_hashes[0] or "").strip().lower() - except Exception: - found_hash = None - found = True - break - - if not found: - continue - - seen_pairs.add((original_url, str(backend_name))) - matched_urls.add(original_url) - display_row = build_table_result_payload( - title="(exists)", - columns=[ - ("Title", "(exists)"), - ("Store", str(backend_name)), - ("Hash", found_hash or ""), - ("URL", original_url), - ], - store=str(backend_name), - hash=found_hash or "", - url=original_url, - ) - match_rows.append(display_row) - continue - - if bulk_mode and bulk_patterns: - bulk_hits: Optional[List[Any]] = None - bulk_limit = min(2000, max(200, len(unique_urls) * 8)) - try: - bulk_hits = backend.search( - "url:*", - limit=bulk_limit, - pattern_hint=bulk_patterns, - ) or [] - except Exception: - try: - bulk_hits = backend.search("url:*", limit=bulk_limit) or [] - except Exception: - bulk_hits = None - - if bulk_hits is not None: - for hit in bulk_hits: - if _timed_out("backend bulk scan"): - return True - if len(match_rows) >= max_rows: - break - url_values = _extract_urls_from_hit(hit, backend, allow_backend_lookup=False) - if not url_values: - continue - - for original_url, needles in url_needles.items(): - if _timed_out("backend bulk scan"): - return True - if len(match_rows) >= max_rows: - break - if (original_url, str(backend_name)) in seen_pairs: - continue - - matched = False - for url_value in url_values: - for needle in (needles or []): - if _match_normalized_url(str(needle or ""), str(url_value or "")): - matched = True - break - if matched: - break - - if not matched: - continue - - seen_pairs.add((original_url, str(backend_name))) - matched_urls.add(original_url) - match_rows.append( - _build_display_row_for_hit(hit, str(backend_name), original_url) - ) - continue - - for original_url, needles in url_needles.items(): - if _timed_out("backend per-url scan"): - return True - if len(match_rows) >= max_rows: - break - if (original_url, str(backend_name)) in seen_pairs: - continue - - display_row = _search_backend_url_hits(backend, str(backend_name), original_url, needles) - if not display_row: - continue - - seen_pairs.add((original_url, str(backend_name))) - matched_urls.add(original_url) - match_rows.append(display_row) - - if not match_rows: - if in_pipeline: - preflight_cache = _load_preflight_cache() - url_dup_cache = preflight_cache.get("url_duplicates") - if not isinstance(url_dup_cache, dict): - url_dup_cache = {} - - cached_urls = url_dup_cache.get("urls") - cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() - - for original_url, needles in url_needles.items(): - cached_set.add(original_url) - for needle in needles or []: - cached_set.add(str(needle)) - - url_dup_cache["urls"] = sorted(cached_set) - preflight_cache["url_duplicates"] = url_dup_cache - _store_preflight_cache(preflight_cache) - _mark_preflight_checked() - return True - - table = Table(f"URL already exists ({len(matched_urls)} url(s))", max_columns=10) - table._interactive(True) - try: - table._perseverance(True) - except Exception: - pass - - for row in match_rows: - table.add_result(row) - - try: - pipeline_context.set_last_result_table_overlay(table, match_rows) - except Exception: - pass - - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm: AbstractContextManager[Any] = nullcontext() - if callable(suspend): - try: - maybe_cm = suspend() - if maybe_cm is not None: - cm = maybe_cm # type: ignore[assignment] - except Exception: - cm = nullcontext() - - auto_confirm_reason: Optional[str] = None - if in_pipeline and stage_ctx is not None: - try: - total_stages = int(getattr(stage_ctx, "total_stages", 0)) - except Exception: - total_stages = 0 - try: - is_last_stage = bool(getattr(stage_ctx, "is_last_stage", False)) - except Exception: - is_last_stage = False - if total_stages > 1 and not is_last_stage and not force_prompt_in_pipeline: - auto_confirm_reason = "pipeline stage (pre-last)" - if auto_confirm_reason is None: - try: - stdin_interactive = bool(sys.stdin and sys.stdin.isatty()) - except Exception: - stdin_interactive = False - if not stdin_interactive: - auto_confirm_reason = "non-interactive stdin" - - answered_yes = True - auto_declined = False - with cm: - get_stderr_console().print(table) - setattr(table, "_rendered_by_cmdlet", True) - if auto_confirm_reason is None: - answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console())) - else: - answered_yes = bool(auto_continue_duplicates) - auto_declined = not answered_yes - if answered_yes: - debug( - f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input." - ) - try: - log( - f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...", - file=sys.stderr, - ) - except Exception: - pass - else: - debug( - f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input." - ) - try: - log( - f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.", - file=sys.stderr, - ) - except Exception: - pass - - if in_pipeline and auto_continue_duplicates: - try: - existing = pipeline_context.load_value("preflight", default=None) - except Exception: - existing = None - preflight_cache: Dict[str, Any] = existing if isinstance(existing, dict) else {} - url_dup_cache = preflight_cache.get("url_duplicates") - if not isinstance(url_dup_cache, dict): - url_dup_cache = {} - url_dup_cache["command"] = str(current_cmd_text or "") - url_dup_cache["continue"] = bool(answered_yes) - cached_urls = url_dup_cache.get("urls") - cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() - for original_url, needles in url_needles.items(): - cached_set.add(original_url) - for needle in needles or []: - cached_set.add(str(needle)) - url_dup_cache["urls"] = sorted(cached_set) - preflight_cache["url_duplicates"] = url_dup_cache - try: - pipeline_context.store_value("preflight", preflight_cache) - except Exception: - pass - - if not answered_yes: - if in_pipeline and not auto_declined: - try: - pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0) - except Exception: - pass - _mark_preflight_checked() - return False - _mark_preflight_checked() - return True - - -def display_and_persist_items( - items: List[Any], - title: str = "Result", - subject: Optional[Any] = None, - display_type: str = "item", - table: Optional[Table] = None, -) -> None: - """Display detail panels and persist items for @N selection. - - This helper function: - 1. Renders individual detail panels for each item - 2. Creates a result table with all items (or uses provided table) - 3. Persists the table so @N selection works across command boundaries - - Args: - items: List of items/dicts to display and make selectable - title: Title for the result table (default: "Result") - subject: Optional subject to associate with the results (usually first item or list) - display_type: Type of display - "item" (default), "tag", or "custom" (no panels) - table: Optional pre-built Table object (if None, creates new Table with title) - """ - if not items: - return - - try: - # Create result table if not provided - if table is None: - table = Table(title=title) - - # Render detail panels for each item (unless display_type is "custom") - if display_type != "custom": - try: - from SYS.rich_display import render_item_details_panel - - # Determine panel title prefix based on display type - panel_prefix = "Tag" if display_type == "tag" else "Item" - - # Render detail panel for each item - for idx, item in enumerate(items, 1): - try: - render_item_details_panel(item, title=f"#{idx} {panel_prefix} Details") - except Exception: - pass - except Exception: - pass - - # Add items to table if not already added by caller - if not getattr(table, "_items_added", False): - for item in items: - table.add_result(item) - - setattr(table, "_rendered_by_cmdlet", True) - - # Persist table for @N selection across command boundaries - publish_result_table(pipeline_context, table, items, subject=subject) - except Exception: - pass diff --git a/cmdlet/_store_utils.py b/cmdlet/_store_utils.py new file mode 100644 index 0000000..1c27263 --- /dev/null +++ b/cmdlet/_store_utils.py @@ -0,0 +1,452 @@ +"""Store and storage operation utilities. + +Contains hash resolution, store batch dispatching, hash query parsing, and +Hydrus metadata fetching helpers used by store-related cmdlets. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from pathlib import Path + +from SYS.logger import log +from ._preflight import get_store_backend + +__all__ = [ + "resolve_hash_for_cmdlet", + "parse_hash_query", + "parse_single_hash_query", + "require_hash_query", + "require_single_hash_query", + "get_hash_for_operation", + "fetch_hydrus_metadata", + "coalesce_hash_value_pairs", + "run_store_hash_value_batches", + "run_store_note_batches", + "collect_store_hash_value_batch", +] + + +def resolve_hash_for_cmdlet( + raw_hash: Optional[str], + raw_path: Optional[str], + override_hash: Optional[str], +) -> Optional[str]: + """Resolve a file hash for note/tag/file cmdlets. + + Shared implementation used by add-note, delete-note, get-note, and similar + cmdlets that need to identify a file by its SHA-256 hash. + + Resolution order: + 1. ``override_hash`` — explicit hash provided via *-query* (highest priority) + 2. ``raw_hash`` — positional hash argument + 3. ``raw_path`` stem — if the filename stem is a 64-char hex string it is + treated directly as the hash (Hydrus-style naming convention) + 4. SHA-256 computed from the file at ``raw_path`` + + Args: + raw_hash: Hash string from positional argument. + raw_path: Filesystem path to the file (may be None). + override_hash: Hash extracted from *-query* (takes precedence). + + Returns: + Normalised 64-char lowercase hex hash, or ``None`` if unresolvable. + """ + from ._tag_utils import normalize_hash + + resolved = normalize_hash(override_hash) if override_hash else normalize_hash(raw_hash) + if resolved: + return resolved + if raw_path: + try: + p = Path(str(raw_path)) + stem = p.stem + if len(stem) == 64 and all(c in "0123456789abcdef" for c in stem.lower()): + return stem.lower() + if p.exists() and p.is_file(): + from SYS.utils import sha256_file as _sha256_file + return _sha256_file(p) + except Exception: + return None + return None + + +def parse_hash_query(query: Optional[str]) -> List[str]: + """Parse a unified query string for `hash:` into normalized SHA256 hashes. + + Supported examples: + - hash:

+ - hash:

,

,

+ - Hash:

+ - hash:{

,

} + + Returns: + List of unique normalized 64-hex SHA256 hashes. + """ + import re + from ._tag_utils import normalize_hash + + q = str(query or "").strip() + if not q: + return [] + + m = re.match(r"^hash(?:es)?\s*:\s*(.+)$", q, flags=re.IGNORECASE) + if not m: + return [] + + rest = (m.group(1) or "").strip() + if rest.startswith("{") and rest.endswith("}"): + rest = rest[1:-1].strip() + if rest.startswith("[") and rest.endswith("]"): + rest = rest[1:-1].strip() + + raw_parts = [p.strip() for p in re.split(r"[\s,]+", rest) if p.strip()] + out: List[str] = [] + for part in raw_parts: + h = normalize_hash(part) + if not h: + continue + if h not in out: + out.append(h) + return out + + +def parse_single_hash_query(query: Optional[str]) -> Optional[str]: + """Parse `hash:` query and require exactly one hash.""" + hashes = parse_hash_query(query) + if len(hashes) != 1: + return None + return hashes[0] + + +def require_hash_query( + query: Optional[str], + error_message: str, + *, + log_file: Any = None, +) -> Tuple[List[str], bool]: + """Parse a multi-hash query and log a caller-provided error on invalid input.""" + hashes = parse_hash_query(query) + if query and not hashes: + kwargs = {"file": log_file} if log_file is not None else {} + log(error_message, **kwargs) + return [], False + return hashes, True + + +def require_single_hash_query( + query: Optional[str], + error_message: str, + *, + log_file: Any = None, +) -> Tuple[Optional[str], bool]: + """Parse a single-hash query and log a caller-provided error on invalid input.""" + query_hash = parse_single_hash_query(query) + if query and not query_hash: + kwargs = {"file": log_file} if log_file is not None else {} + log(error_message, **kwargs) + return None, False + return query_hash, True + + +def get_hash_for_operation( + override_hash: Optional[str], + result: Any, + field_name: str = "hash", +) -> Optional[str]: + """Get normalized hash from override or result object, consolidating common pattern. + + Eliminates repeated pattern: normalize_hash(override) if override else normalize_hash(get_field(result, ...)) + + Args: + override_hash: Hash passed as command argument (takes precedence) + result: Object containing hash field (fallback) + field_name: Name of hash field in result object (default: "hash") + + Returns: + Normalized hash string, or None if neither override nor result provides valid hash + """ + from ._tag_utils import normalize_hash + from ._pipeobject_utils import get_field + + if override_hash: + return normalize_hash(override_hash) + hash_value = ( + get_field(result, + field_name) or getattr(result, + field_name, + None) or getattr(result, + "hash", + None) + ) + return normalize_hash(hash_value) + + +def fetch_hydrus_metadata( + config: Any, + hash_hex: str, + *, + store_name: Optional[str] = None, + hydrus_client: Any = None, + **kwargs, +) -> tuple[Optional[Dict[str, Any]], Optional[int]]: + """Fetch metadata from Hydrus for a given hash, consolidating common fetch pattern. + + Eliminates repeated boilerplate: client initialization, error handling, metadata extraction. + + Args: + config: Configuration object used to resolve the Hydrus plugin/store + hash_hex: File hash to fetch metadata for + store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client. + hydrus_client: Optional explicit Hydrus client. When provided, takes precedence. + **kwargs: Additional arguments to pass to client.fetch_file_metadata() + Common: include_service_keys_to_tags, include_notes, include_file_url, include_duration, etc. + + Returns: + Tuple of (metadata_dict, error_code) + - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable + - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute()) + """ + client = hydrus_client + hydrus_provider = None + try: + from PluginCore.registry import get_plugin + + hydrus_provider = get_plugin("hydrusnetwork", config) + except Exception: + hydrus_provider = None + + if client is None: + if hydrus_provider is not None: + try: + client = hydrus_provider.get_client( + store_name=store_name if store_name else None, + allow_default=not bool(store_name), + ) + except Exception as exc: + if store_name: + log(f"Hydrus client unavailable for store '{store_name}': {exc}") + else: + log(f"Hydrus client unavailable: {exc}") + client = None + if client is None and store_name: + log(f"Hydrus client unavailable for store '{store_name}'") + return None, 1 + if client is None and hydrus_provider is None: + log("Hydrus provider unavailable") + return None, 1 + + if hydrus_provider is not None: + try: + metadata = hydrus_provider.fetch_metadata( + hash_hex, + store_name=store_name if store_name else None, + **kwargs, + ) + except Exception as exc: + log(f"Hydrus metadata fetch failed: {exc}") + return None, 1 + if isinstance(metadata, dict): + return metadata, 0 + if client is None: + if store_name: + log(f"Hydrus client unavailable for store '{store_name}'") + else: + log("Hydrus metadata unavailable") + return None, 1 + + try: + payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs) + except Exception as exc: + log(f"Hydrus metadata fetch failed: {exc}") + return None, 1 + + items = payload.get("metadata") if isinstance(payload, dict) else None + meta = items[0] if ( + isinstance(items, list) and items and isinstance(items[0], dict) + ) else None + + return meta, 0 + + +def coalesce_hash_value_pairs( + pairs: Sequence[Tuple[str, Sequence[str]]], +) -> List[Tuple[str, List[str]]]: + """Merge duplicate hash/value pairs while preserving first-seen value order.""" + merged: Dict[str, List[str]] = {} + for hash_value, values in pairs: + normalized_hash = str(hash_value or "").strip() + if not normalized_hash: + continue + bucket = merged.setdefault(normalized_hash, []) + seen = set(bucket) + for value in values or []: + text = str(value or "").strip() + if not text or text in seen: + continue + seen.add(text) + bucket.append(text) + return [(hash_value, items) for hash_value, items in merged.items() if items] + + +def run_store_hash_value_batches( + config: Optional[Dict[str, Any]], + batch: Dict[str, List[Tuple[str, Sequence[str]]]], + *, + bulk_method_name: str, + single_method_name: str, + store_registry: Any = None, + suppress_debug: bool = False, + pass_config_to_bulk: bool = True, + pass_config_to_single: bool = True, +) -> Tuple[Any, List[Tuple[str, int, int]]]: + """Dispatch grouped hash/value batches across stores. + + Returns ``(store_registry, stats)`` where ``stats`` contains + ``(store_name, item_count, value_count)`` for each dispatched store. + Missing stores are skipped so callers can preserve existing warning behavior. + """ + registry = store_registry + stats: List[Tuple[str, int, int]] = [] + for store_name, pairs in batch.items(): + backend, registry, _exc = get_store_backend( + config, + store_name, + store_registry=registry, + suppress_debug=suppress_debug, + ) + if backend is None: + continue + + bulk_pairs = coalesce_hash_value_pairs(pairs) + if not bulk_pairs: + continue + + bulk_fn = getattr(backend, bulk_method_name, None) + if callable(bulk_fn): + if pass_config_to_bulk: + bulk_fn(bulk_pairs, config=config) + else: + bulk_fn(bulk_pairs) + else: + single_fn = getattr(backend, single_method_name) + for hash_value, values in bulk_pairs: + if pass_config_to_single: + single_fn(hash_value, values, config=config) + else: + single_fn(hash_value, values) + + stats.append( + ( + store_name, + len(bulk_pairs), + sum(len(values or []) for _hash_value, values in bulk_pairs), + ) + ) + + return registry, stats + + +def run_store_note_batches( + config: Optional[Dict[str, Any]], + batch: Dict[str, List[Tuple[str, str, str]]], + *, + store_registry: Any = None, + suppress_debug: bool = False, + on_store_error: Optional[Callable[[str, Exception], None]] = None, + on_unsupported_store: Optional[Callable[[str], None]] = None, + on_item_error: Optional[Callable[[str, str, str, Exception], None]] = None, +) -> Tuple[Any, int]: + """Dispatch grouped note writes across stores while preserving item-level errors.""" + registry = store_registry + success_count = 0 + for store_name, items in batch.items(): + backend, registry, exc = get_store_backend( + config, + store_name, + store_registry=registry, + suppress_debug=suppress_debug, + ) + if backend is None: + if on_store_error is not None and exc is not None: + on_store_error(store_name, exc) + continue + supports_note_capability = getattr(backend, "supports_note_association", None) + if supports_note_capability is None: + supports_note_capability = hasattr(backend, "set_note") + if not bool(supports_note_capability): + if on_unsupported_store is not None: + on_unsupported_store(store_name) + continue + if not hasattr(backend, "set_note"): + if on_unsupported_store is not None: + on_unsupported_store(store_name) + continue + + for hash_value, note_name, note_text in items: + try: + if backend.set_note(hash_value, note_name, note_text, config=config): + success_count += 1 + except Exception as item_exc: + if on_item_error is not None: + on_item_error(store_name, hash_value, note_name, item_exc) + + return registry, success_count + + +def collect_store_hash_value_batch( + items: Sequence[Any], + *, + store_registry: Any, + value_resolver: Callable[[Any], Optional[Sequence[str]]], + override_hash: Optional[str] = None, + override_store: Optional[str] = None, + on_warning: Optional[Callable[[str], None]] = None, +) -> Tuple[Dict[str, List[Tuple[str, List[str]]]], List[Any]]: + """Collect validated store/hash/value batches while preserving passthrough items.""" + from ._tag_utils import normalize_hash + from ._pipeobject_utils import get_field as _get_field + + batch: Dict[str, List[Tuple[str, List[str]]]] = {} + pass_through: List[Any] = [] + + for item in items: + pass_through.append(item) + + raw_hash = override_hash or _get_field(item, "hash") + raw_store = override_store or _get_field(item, "store") + if not raw_hash or not raw_store: + if on_warning is not None: + on_warning("Item missing hash/store; skipping") + continue + + normalized = normalize_hash(raw_hash) + if not normalized: + if on_warning is not None: + on_warning("Item has invalid hash; skipping") + continue + + store_text = str(raw_store).strip() + if not store_text: + if on_warning is not None: + on_warning("Item has empty store; skipping") + continue + + try: + is_available = bool(store_registry.is_available(store_text)) + except Exception: + is_available = False + if not is_available: + if on_warning is not None: + on_warning(f"Store '{store_text}' not configured; skipping") + continue + + values = [str(value).strip() for value in (value_resolver(item) or []) if str(value).strip()] + if not values: + continue + + batch.setdefault(store_text, []).append((normalized, values)) + + return batch, pass_through + diff --git a/cmdlet/_tag_utils.py b/cmdlet/_tag_utils.py new file mode 100644 index 0000000..c130fb0 --- /dev/null +++ b/cmdlet/_tag_utils.py @@ -0,0 +1,954 @@ +"""Tag-related utilities: parsing, normalization, template rendering, and extraction. + +Covers the full tag lifecycle: +- Parsing raw tag arguments from command-line tokens +- Normalizing hash strings for tag identity +- Expanding tag group references from JSON config files +- Rendering tag value templates with ``#(placeholder)`` and ```` syntax +- Extracting tags, titles, and URLs from result objects (PipeObject/dict) +- Manipulating tag lists (namespace collapse, preferred titles, relationships) +""" + +from __future__ import annotations + +import json +import re +import sys +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +from SYS import models +from SYS.logger import log +from ._pipeobject_utils import merge_sequences + +__all__ = [ + "set_tag_groups_path", + "normalize_hash", + "looks_like_hash", + "parse_tag_arguments", + "render_tag_value_templates", + "build_tag_value_lookup", + "_add_tag_values_to_lookup", + "expand_tag_groups", + "first_title_tag", + "apply_preferred_title", + "collapse_namespace_tags", + "collect_relationship_labels", + "extract_tag_from_result", + "extract_title_from_result", + "extract_url_from_result", +] + +TAG_GROUPS_PATH: Optional[Path] = None + +_TAG_GROUPS_CACHE: Optional[Dict[str, List[str]]] = None +_TAG_GROUPS_MTIME: Optional[float] = None + +_TAG_VALUE_TEMPLATE_RE = re.compile(r"#\(([^)]+)\)") +_TAG_VALUE_FUNCTION_RE = re.compile(r"<([a-zA-Z_][a-zA-Z0-9_-]*)\((.*?)\)>") + + +def set_tag_groups_path(path: Path) -> None: + """Set the path to the tag groups JSON file.""" + global TAG_GROUPS_PATH + TAG_GROUPS_PATH = path + + +@lru_cache(maxsize=4096) +def _normalize_hash_cached(hash_hex: str) -> Optional[str]: + text = hash_hex.strip().lower() + if not text: + return None + if len(text) != 64: + return None + if not all(ch in "0123456789abcdef" for ch in text): + return None + return text + + +def normalize_hash(hash_hex: Optional[str]) -> Optional[str]: + """Normalize a hash string to lowercase, or return None if invalid. + + Args: + hash_hex: String that should be a hex hash + + Returns: + Lowercase hash string, or None if input is not a string or is empty + """ + if not isinstance(hash_hex, str): + return None + return _normalize_hash_cached(hash_hex) + + +def looks_like_hash(candidate: Optional[str]) -> bool: + """Check if a string looks like a SHA256 hash (64 hex chars). + + Args: + candidate: String to test + + Returns: + True if the string is 64 lowercase hex characters + """ + if not isinstance(candidate, str): + return False + text = candidate.strip().lower() + return len(text) == 64 and all(ch in "0123456789abcdef" for ch in text) + + +def _normalize_tag_value_template_name(value: Any) -> str: + text = str(value or "").strip().lower() + if not text: + return "" + try: + text = re.sub(r"\s+", " ", text).strip() + except Exception: + text = " ".join(text.split()) + return text + + +def _tag_value_template_keys(value: Any) -> list[str]: + normalized = _normalize_tag_value_template_name(value) + if not normalized: + return [] + + keys = [normalized] + + trimmed_hash = re.sub(r"\s*#+\s*$", "", normalized).strip() + if trimmed_hash and trimmed_hash not in keys: + keys.append(trimmed_hash) + + return keys + + +def _add_tag_values_to_lookup(lookup: Dict[str, List[str]], tag_text: Any) -> None: + text = str(tag_text or "").strip() + if not text or ":" not in text: + return + if _TAG_VALUE_TEMPLATE_RE.search(text) or _TAG_VALUE_FUNCTION_RE.search(text): + return + + namespace, value = text.split(":", 1) + value_text = str(value or "").strip() + if not value_text: + return + + for key in _tag_value_template_keys(namespace): + values = lookup.setdefault(key, []) + if value_text not in values: + values.append(value_text) + + +def build_tag_value_lookup( + tags: Optional[Iterable[Any]], + *, + result: Any = None, +) -> Dict[str, List[str]]: + """Build a placeholder lookup from existing tags and lightweight result fields. + + Placeholder lookups use ``#(namespace)`` syntax. Namespace matching is + case-insensitive and trims repeated whitespace. A trailing ``#`` in the + placeholder is ignored so inputs like ``#(track #)`` can resolve ``track:9``. + """ + + lookup: Dict[str, List[str]] = {} + for tag in tags or []: + _add_tag_values_to_lookup(lookup, tag) + + title_text = extract_title_from_result(result) + if title_text: + _add_tag_values_to_lookup(lookup, f"title:{title_text}") + + return lookup + + +def _split_tag_value_function_args(value: Any) -> list[str]: + text = str(value or "") + args: list[str] = [] + current: list[str] = [] + depth = 0 + quote: Optional[str] = None + escape = False + + for ch in text: + if escape: + current.append(ch) + escape = False + continue + if ch == "\\": + current.append(ch) + escape = True + continue + if quote: + current.append(ch) + if ch == quote: + quote = None + continue + if ch in {"'", '"'}: + current.append(ch) + quote = ch + continue + if ch in {"(", "[", "{"}: + depth += 1 + current.append(ch) + continue + if ch in {")", "]", "}"}: + depth = max(0, depth - 1) + current.append(ch) + continue + if ch == "," and depth == 0: + args.append("".join(current).strip()) + current = [] + continue + current.append(ch) + + tail = "".join(current).strip() + if tail or args: + args.append(tail) + return args + + +def _strip_tag_value_function_arg(value: Any) -> str: + text = str(value or "").strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}: + return text[1:-1] + return text + + +def _padding_width_from_spec(value: Any) -> Optional[int]: + spec = _strip_tag_value_function_arg(value) + if not spec: + return None + if re.fullmatch(r"0+", spec): + return len(spec) + if spec.isdigit(): + try: + width = int(spec) + except Exception: + return None + return width if width > 0 else None + return None + + +def _replace_tag_value_placeholders( + value: Any, + lookup: Dict[str, List[str]], + *, + preserve_unresolved: bool, +) -> tuple[str, bool]: + text = str(value or "") + unresolved = False + + def _replace(match: re.Match[str]) -> str: + nonlocal unresolved + keys = _tag_value_template_keys(match.group(1) or "") + values: List[str] = [] + for key in keys: + for candidate in lookup.get(key, []): + if candidate not in values: + values.append(candidate) + if not values: + unresolved = True + return match.group(0) if preserve_unresolved else "" + return ", ".join(values) + + return _TAG_VALUE_TEMPLATE_RE.sub(_replace, text), unresolved + + +def _coerce_tag_value_integer(value: Any) -> Optional[int]: + text = _strip_tag_value_function_arg(value) + if not text: + return None + if not re.fullmatch(r"[+-]?\d+", text): + return None + try: + return int(text) + except Exception: + return None + + +def _apply_tag_value_function( + name: str, + args: Sequence[str], + *, + lookup: Dict[str, List[str]], +) -> Optional[str]: + func = str(name or "").strip().lower() + + resolved_values: list[str] = [] + unresolved_flags: list[bool] = [] + for arg in args: + rendered, unresolved = _replace_tag_value_placeholders( + arg, + lookup, + preserve_unresolved=True, + ) + resolved_values.append(_strip_tag_value_function_arg(rendered)) + unresolved_flags.append(unresolved) + + if func in {"padding", "pad", "zfill"}: + if len(resolved_values) != 2 or any(unresolved_flags): + return None + width = _padding_width_from_spec(resolved_values[0]) + if width is None: + return None + return str(resolved_values[1]).zfill(width) + + if func == "default": + if len(resolved_values) != 2: + return None + primary = resolved_values[0] + fallback = resolved_values[1] + if not unresolved_flags[0] and str(primary).strip(): + return str(primary) + if unresolved_flags[1]: + return None + return str(fallback) + + if func == "replace": + if len(resolved_values) != 3 or any(unresolved_flags): + return None + return str(resolved_values[0]).replace( + str(resolved_values[1]), + str(resolved_values[2]), + ) + + if func in {"increment", "inc", "add"}: + if len(resolved_values) not in {1, 2}: + return None + if unresolved_flags[0]: + return None + base_value = _coerce_tag_value_integer(resolved_values[0]) + if base_value is None: + return None + step_value = 1 + if len(resolved_values) == 2: + if unresolved_flags[1]: + return None + parsed_step = _coerce_tag_value_integer(resolved_values[1]) + if parsed_step is None: + return None + step_value = parsed_step + return str(base_value + step_value) + + return None + + +def _render_tag_value_function_templates( + value: Any, + *, + lookup: Dict[str, List[str]], +) -> tuple[str, bool]: + text = str(value or "") + unresolved = False + + def _replace(match: re.Match[str]) -> str: + nonlocal unresolved + func_name = match.group(1) or "" + func_args = _split_tag_value_function_args(match.group(2) or "") + rendered = _apply_tag_value_function( + func_name, + func_args, + lookup=lookup, + ) + if rendered is None: + unresolved = True + return match.group(0) + return rendered + + previous = None + rendered = text + while previous != rendered and _TAG_VALUE_FUNCTION_RE.search(rendered): + previous = rendered + rendered = _TAG_VALUE_FUNCTION_RE.sub(_replace, rendered) + if unresolved: + break + return rendered, unresolved + + +def render_tag_value_templates( + tags: Sequence[Any], + *, + existing_tags: Optional[Iterable[Any]] = None, + result: Any = None, +) -> tuple[list[str], list[str]]: + """Resolve ``#(namespace)`` placeholders and ```` functions. + + Returns ``(resolved_tags, unresolved_templates)``. Tags whose placeholders + cannot be fully resolved are omitted from ``resolved_tags`` and returned in + ``unresolved_templates`` so callers can warn or summarize skipped items. + + Currently supported transforms: + - ```` or ```` for zero-padding + - ```` to fall back when a placeholder is missing + - ```` for simple substring replacement + - ```` for integer arithmetic + """ + + entries: list[dict[str, Any]] = [] + lookup = build_tag_value_lookup(existing_tags, result=result) + + for raw_tag in tags or []: + text = str(raw_tag or "").strip() + if not text: + continue + has_template = bool( + _TAG_VALUE_TEMPLATE_RE.search(text) + or _TAG_VALUE_FUNCTION_RE.search(text) + ) + entry = { + "raw": text, + "resolved": None, + "has_template": has_template, + } + if not has_template: + entry["resolved"] = text + _add_tag_values_to_lookup(lookup, text) + entries.append(entry) + + progress = True + while progress: + progress = False + for entry in entries: + if entry["resolved"] is not None or not entry["has_template"]: + continue + + rendered, unresolved = _replace_tag_value_placeholders( + entry["raw"], + lookup, + preserve_unresolved=bool(_TAG_VALUE_FUNCTION_RE.search(str(entry["raw"]))), + ) + + rendered, function_unresolved = _render_tag_value_function_templates( + rendered, + lookup=lookup, + ) + if function_unresolved: + continue + + if unresolved and _TAG_VALUE_TEMPLATE_RE.search(rendered): + continue + + rendered = rendered.strip() + if not rendered: + entry["resolved"] = "" + progress = True + continue + + entry["resolved"] = rendered + _add_tag_values_to_lookup(lookup, rendered) + progress = True + + resolved_tags = merge_sequences( + [entry["resolved"] for entry in entries if isinstance(entry.get("resolved"), str) and entry.get("resolved")], + case_sensitive=True, + ) + unresolved_templates = [ + str(entry["raw"]) + for entry in entries + if entry["has_template"] and not entry.get("resolved") + ] + return resolved_tags, unresolved_templates + + +def _normalize_tag_group_entry(value: Any) -> Optional[str]: + """Internal: Normalize a single tag group entry.""" + if not isinstance(value, str): + value = str(value) + text = value.strip() + return text or None + + +def _load_tag_groups() -> Dict[str, List[str]]: + """Load tag group definitions from JSON file with caching.""" + global _TAG_GROUPS_CACHE, _TAG_GROUPS_MTIME, TAG_GROUPS_PATH + + if TAG_GROUPS_PATH is None: + try: + script_dir = Path(__file__).parent.parent + + candidate = script_dir / "adjective.json" + if candidate.exists(): + TAG_GROUPS_PATH = candidate + else: + candidate = script_dir / "helper" / "adjective.json" + if candidate.exists(): + TAG_GROUPS_PATH = candidate + except Exception: + pass + + if TAG_GROUPS_PATH is None: + return {} + + path = TAG_GROUPS_PATH + try: + stat_result = path.stat() + except FileNotFoundError: + _TAG_GROUPS_CACHE = {} + _TAG_GROUPS_MTIME = None + return {} + except OSError as exc: + log(f"Failed to read tag groups: {exc}", file=sys.stderr) + _TAG_GROUPS_CACHE = {} + _TAG_GROUPS_MTIME = None + return {} + + mtime = stat_result.st_mtime + if _TAG_GROUPS_CACHE is not None and _TAG_GROUPS_MTIME == mtime: + return _TAG_GROUPS_CACHE + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + log(f"Invalid tag group JSON ({path}): {exc}", file=sys.stderr) + _TAG_GROUPS_CACHE = {} + _TAG_GROUPS_MTIME = mtime + return {} + + groups: Dict[str, List[str]] = {} + if isinstance(payload, dict): + for key, value in payload.items(): + if not isinstance(key, str): + continue + name = key.strip().lower() + if not name: + continue + members: List[str] = [] + if isinstance(value, list): + for entry in value: + normalized = _normalize_tag_group_entry(entry) + if normalized: + members.append(normalized) + elif isinstance(value, str): + normalized = _normalize_tag_group_entry(value) + if normalized: + members.extend( + token.strip() for token in normalized.split(",") + if token.strip() + ) + if members: + groups[name] = members + + _TAG_GROUPS_CACHE = groups + _TAG_GROUPS_MTIME = mtime + return groups + + +def expand_tag_groups(raw_tags: Iterable[str]) -> List[str]: + """Expand tag group references (e.g., {my_group}) into member tags. + + Tag groups are defined in JSON and can be nested. Groups are referenced + with curly braces: {group_name}. + + Args: + raw_tags: Sequence of tag strings, some may reference groups like "{group_name}" + + Returns: + List of expanded tags with group references replaced + """ + groups = _load_tag_groups() + if not groups: + return [tag for tag in raw_tags if isinstance(tag, str) and tag.strip()] + + def _expand(tokens: Iterable[str], seen: Set[str]) -> List[str]: + result: List[str] = [] + for token in tokens: + if not isinstance(token, str): + continue + candidate = token.strip() + if not candidate: + continue + if candidate.startswith("{") and candidate.endswith("}") and len(candidate) > 2: + name = candidate[1:-1].strip().lower() + if not name: + continue + if name in seen: + log( + f"Tag group recursion detected for {{{name}}}; skipping", + file=sys.stderr, + ) + continue + members = groups.get(name) + if not members: + log(f"Unknown tag group {{{name}}}", file=sys.stderr) + result.append(candidate) + continue + result.extend(_expand(members, seen | {name})) + else: + result.append(candidate) + return result + + return _expand(raw_tags, set()) + + +def parse_tag_arguments(arguments: Sequence[str]) -> List[str]: + """Parse tag arguments from command line tokens. + + - Supports comma-separated tags. + - Supports pipe namespace shorthand: "artist:A|B|C" -> artist:A, artist:B, artist:C. + + Args: + arguments: Sequence of argument strings + + Returns: + List of normalized tag strings (empty strings filtered out) + """ + + def _split_top_level_commas(text: str) -> List[str]: + segments: List[str] = [] + current: List[str] = [] + paren_depth = 0 + angle_depth = 0 + quote: Optional[str] = None + escape = False + + for ch in text: + if escape: + current.append(ch) + escape = False + continue + if ch == "\\": + current.append(ch) + escape = True + continue + if quote: + current.append(ch) + if ch == quote: + quote = None + continue + if ch in {"'", '"'}: + current.append(ch) + quote = ch + continue + if ch == "(": + paren_depth += 1 + current.append(ch) + continue + if ch == ")": + paren_depth = max(0, paren_depth - 1) + current.append(ch) + continue + if ch == "<": + angle_depth += 1 + current.append(ch) + continue + if ch == ">": + angle_depth = max(0, angle_depth - 1) + current.append(ch) + continue + if ch == "," and paren_depth == 0 and angle_depth == 0: + segments.append("".join(current).strip()) + current = [] + continue + current.append(ch) + + tail = "".join(current).strip() + if tail or segments: + segments.append(tail) + return segments + + def _expand_pipe_namespace(text: str) -> List[str]: + parts = text.split("|") + expanded: List[str] = [] + last_ns: Optional[str] = None + for part in parts: + segment = part.strip() + if not segment: + continue + if ":" in segment: + ns, val = segment.split(":", 1) + ns = ns.strip() + val = val.strip() + last_ns = ns or last_ns + if last_ns and val: + expanded.append(f"{last_ns}:{val}") + elif ns or val: + expanded.append(f"{ns}:{val}".strip(":")) + else: + if last_ns: + expanded.append(f"{last_ns}:{segment}") + else: + expanded.append(segment) + return expanded + + tags: List[str] = [] + for argument in arguments: + for token in _split_top_level_commas(str(argument)): + text = token.strip() + if not text: + continue + pipe_expanded = _expand_pipe_namespace(text) + for entry in pipe_expanded: + candidate = entry.strip() + if not candidate: + continue + if ":" in candidate: + ns, val = candidate.split(":", 1) + ns = ns.strip() + val = val.strip() + candidate = f"{ns}:{val}" if ns or val else "" + if candidate: + tags.append(candidate) + return tags + + +def first_title_tag(source: Optional[Iterable[str]]) -> Optional[str]: + """Find the first tag starting with "title:" in a collection. + + Args: + source: Iterable of tag strings + + Returns: + First title: tag found, or None + """ + if not source: + return None + for item in source: + if not isinstance(item, str): + continue + candidate = item.strip() + if candidate and candidate.lower().startswith("title:"): + return candidate + return None + + +def apply_preferred_title(tags: List[str], preferred: Optional[str]) -> List[str]: + """Replace any title: tags with a preferred title tag. + + Args: + tags: List of tags (may contain multiple "title:" entries) + preferred: Preferred title tag to use (full "title: ..." format) + + Returns: + List with old title tags removed and preferred title added (at most once) + """ + if not preferred: + return tags + preferred_clean = preferred.strip() + if not preferred_clean: + return tags + preferred_lower = preferred_clean.lower() + filtered: List[str] = [] + has_preferred = False + for tag in tags: + candidate = tag.strip() + if not candidate: + continue + if candidate.lower().startswith("title:"): + if candidate.lower() == preferred_lower: + if not has_preferred: + filtered.append(candidate) + has_preferred = True + continue + filtered.append(candidate) + if not has_preferred: + filtered.append(preferred_clean) + return filtered + + +def collapse_namespace_tags( + tags: Optional[Iterable[Any]], + namespace: str, + prefer: str = "last", +) -> list[str]: + """Reduce tags so only one entry for a given namespace remains. + + Keeps either the first or last occurrence (default last) while preserving overall order + for non-matching tags. Useful for ensuring a single title: tag. + """ + if not tags: + return [] + ns = str(namespace or "").strip().lower() + if not ns: + return list(tags) if isinstance(tags, list) else list(tags) + + prefer_last = str(prefer or "last").lower() != "first" + ns_prefix = ns + ":" + + items = list(tags) + if prefer_last: + kept: list[str] = [] + seen_ns = False + for tag in reversed(items): + text = str(tag) + if text.lower().startswith(ns_prefix): + if seen_ns: + continue + seen_ns = True + kept.append(text) + kept.reverse() + return kept + else: + kept_ns = False + result: list[str] = [] + for tag in items: + text = str(tag) + if text.lower().startswith(ns_prefix): + if kept_ns: + continue + kept_ns = True + result.append(text) + return result + + +def collect_relationship_labels( + payload: Any, + label_stack: List[str] | None = None, + mapping: Dict[str, str] | None = None, +) -> Dict[str, str]: + """Recursively extract hash-to-label mappings from nested relationship data. + + Walks through nested dicts/lists looking for sha256-like strings (64 hex chars) + and builds a mapping from hash to its path in the structure. + + Example: + data = { + "duplicates": [ + "abc123...", # Will be mapped to "duplicates" + {"type": "related", "items": ["def456..."]} # Will be mapped to "duplicates / type / items" + ] + } + result = collect_relationship_labels(data) + # result = {"abc123...": "duplicates", "def456...": "duplicates / type / items"} + + Args: + payload: Nested data structure (dict, list, string, etc.) + label_stack: Internal use - tracks path during recursion + mapping: Internal use - accumulates hash->label mappings + + Returns: + Dict mapping hash strings to their path labels + """ + if label_stack is None: + label_stack = [] + if mapping is None: + mapping = {} + + if isinstance(payload, dict): + for key, value in payload.items(): + next_stack = label_stack + if isinstance(key, str) and key: + formatted = key.replace("_", " ").strip() + next_stack = label_stack + [formatted] + collect_relationship_labels(value, next_stack, mapping) + elif isinstance(payload, (list, tuple, set)): + for value in payload: + collect_relationship_labels(value, label_stack, mapping) + elif isinstance(payload, str) and looks_like_hash(payload): + hash_value = payload.lower() + if label_stack: + label = " / ".join(item for item in label_stack if item) + else: + label = "related" + mapping.setdefault(hash_value, label) + + return mapping + + +def extract_tag_from_result(result: Any) -> list[str]: + """Extract all tags from a result dict or PipeObject. + + Handles mixed types (lists, sets, strings) and various field names. + """ + tag: list[str] = [] + + def _extend(candidate: Any) -> None: + if not candidate: + return + if isinstance(candidate, (list, set, tuple)): + tag.extend(str(t) for t in candidate if t is not None) + elif isinstance(candidate, str): + tag.append(candidate) + + if isinstance(result, models.PipeObject): + tag.extend(result.tag or []) + if isinstance(result.extra, dict): + _extend(result.extra.get("tag")) + if isinstance(result.metadata, dict): + _extend(result.metadata.get("tag")) + _extend(result.metadata.get("tags")) + elif hasattr(result, "tag"): + _extend(getattr(result, "tag")) + + if isinstance(result, dict): + _extend(result.get("tag")) + _extend(result.get("tags")) + + extra = result.get("extra") + if isinstance(extra, dict): + _extend(extra.get("tag")) + _extend(extra.get("tags")) + + fm = result.get("full_metadata") or result.get("metadata") + if isinstance(fm, dict): + _extend(fm.get("tag")) + _extend(fm.get("tags")) + + return merge_sequences(tag, case_sensitive=True) + + +def extract_title_from_result(result: Any) -> Optional[str]: + """Extract the title from a result dict or PipeObject.""" + if isinstance(result, models.PipeObject): + return result.title + elif hasattr(result, "title"): + return getattr(result, "title") + elif isinstance(result, dict): + return result.get("title") + return None + + +def extract_url_from_result(result: Any) -> list[str]: + """Extract all unique URLs from a result dict or PipeObject. + + Handles mixed types (lists, strings) and various field names (url, source_url, webpage_url). + Centralizes extraction logic for cmdlets like download-file, add-file, get-url. + """ + url: list[str] = [] + + def _extend(candidate: Any) -> None: + if not candidate: + return + if isinstance(candidate, list): + url.extend(candidate) + elif isinstance(candidate, str): + url.append(candidate) + + if isinstance(result, models.PipeObject): + _extend(result.url) + _extend(result.source_url) + if isinstance(result.extra, dict): + _extend(result.extra.get("url")) + _extend(result.extra.get("source_url")) + if isinstance(result.metadata, dict): + _extend(result.metadata.get("url")) + _extend(result.metadata.get("source_url")) + _extend(result.metadata.get("webpage_url")) + if isinstance(getattr(result, "full_metadata", None), dict): + fm = getattr(result, "full_metadata", None) + if isinstance(fm, dict): + _extend(fm.get("url")) + _extend(fm.get("source_url")) + _extend(fm.get("webpage_url")) + + elif hasattr(result, "url") or hasattr(result, "source_url"): + _extend(getattr(result, "url", None)) + _extend(getattr(result, "source_url", None)) + + if isinstance(result, dict): + _extend(result.get("url")) + _extend(result.get("source_url")) + _extend(result.get("webpage_url")) + + extra = result.get("extra") + if isinstance(extra, dict): + _extend(extra.get("url")) + + fm = result.get("full_metadata") or result.get("metadata") + if isinstance(fm, dict): + _extend(fm.get("url")) + _extend(fm.get("source_url")) + _extend(fm.get("webpage_url")) + + from SYS.metadata import normalize_urls + return normalize_urls(url) diff --git a/cmdlet/_template_utils.py b/cmdlet/_template_utils.py new file mode 100644 index 0000000..4d18980 --- /dev/null +++ b/cmdlet/_template_utils.py @@ -0,0 +1,44 @@ +"""Template rendering and pipeline preview utilities. + +Contains helpers for constructing pipeline progress previews and any +template-related utilities that don't belong in tag-specific modules. +""" + +from __future__ import annotations + +from typing import Any, List, Sequence + +from SYS.item_accessors import get_field + +__all__ = [ + "build_pipeline_preview", +] + + +def build_pipeline_preview(raw_urls: Sequence[str], piped_items: Sequence[Any]) -> List[str]: + """Construct a short preview list for pipeline/cmdlet progress UI.""" + preview: List[str] = [] + + try: + for u in (raw_urls or [])[:3]: + if u: + preview.append(str(u)) + except Exception: + pass + + if len(preview) < 5: + try: + for item in (piped_items or [])[:5]: + if len(preview) >= 5: + break + title = get_field(item, "title") or get_field(item, "target") or "Piped item" + preview.append(str(title)) + except Exception: + pass + + if not preview: + total = len(raw_urls or []) + len(piped_items or []) + if total: + preview.append(f"Processing {total} item(s)...") + + return preview diff --git a/cmdlet/_url_utils.py b/cmdlet/_url_utils.py new file mode 100644 index 0000000..6db0155 --- /dev/null +++ b/cmdlet/_url_utils.py @@ -0,0 +1,941 @@ +"""URL existence checks, URL merging/removal helpers, and URL preflight logic. + +Contains ``check_url_exists_in_storage`` with all its nested closures that +perform bulk preflight checks across storage backends to determine whether +URLs already exist before downloading. +""" + +from __future__ import annotations + +import re +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from SYS.logger import log, debug, debug_panel +from SYS import pipeline as pipeline_context +from SYS.item_accessors import get_field +from SYS.payload_builders import build_table_result_payload +from SYS.result_table import Table +from SYS.rich_display import stderr_console as get_stderr_console +from rich.prompt import Confirm +from contextlib import AbstractContextManager, nullcontext + +__all__ = [ + "check_url_exists_in_storage", + "merge_urls", + "remove_urls", + "set_item_urls", + "register_url_with_local_library", +] + + +def register_url_with_local_library( + pipe_obj: Any, + config: Dict[str, Any], +) -> bool: + """Register url with a file in the local library database. + + This is called automatically by download cmdlet to ensure url are persisted + without requiring a separate add-url step in the pipeline. + + Args: + pipe_obj: PipeObject with path and url + config: Config dict containing local library path + + Returns: + True if url were registered, False otherwise + """ + return False + + +def merge_urls(existing: Any, incoming: Sequence[Any]) -> list[str]: + """Merge URL values into a normalized, de-duplicated list.""" + from SYS.metadata import normalize_urls + + merged: list[str] = [] + for value in normalize_urls(existing): + if value not in merged: + merged.append(value) + for value in normalize_urls(list(incoming or [])): + if value not in merged: + merged.append(value) + return merged + + +def remove_urls(existing: Any, remove: Sequence[Any]) -> list[str]: + """Remove URL values from an existing URL field and return survivors.""" + from SYS.metadata import normalize_urls + + current = normalize_urls(existing) + remove_set = {value for value in normalize_urls(list(remove or [])) if value} + if not remove_set: + return current + return [value for value in current if value not in remove_set] + + +def set_item_urls(item: Any, urls: Sequence[Any]) -> None: + """Persist normalized URL values back onto a dict/object result item.""" + normalized = merge_urls([], list(urls or [])) + payload: Any = normalized[0] if len(normalized) == 1 else list(normalized) + + try: + if isinstance(item, dict): + item["url"] = payload + return + if hasattr(item, "url"): + setattr(item, "url", payload) + except Exception: + return + + +def check_url_exists_in_storage( + urls: Sequence[str], + storage: Any, + hydrus_available: bool, + final_output_dir: Optional[Path] = None, + *, + auto_continue_duplicates: bool = True, + force_prompt_in_pipeline: bool = False, +) -> bool: + """Pre-flight check to see if URLs already exist in storage. + + Args: + urls: List of URLs to check + storage: The storage interface + hydrus_available: Whether Hydrus is available + final_output_dir: Final output directory (to skip if same as storage) + + Returns: + True if check passed (user said yes or no dups), False if user said no (stop). + """ + if storage is None: + debug("Bulk URL preflight skipped: storage unavailable") + return True + + try: + current_cmd_text = pipeline_context.get_current_command_text("") + except Exception: + current_cmd_text = "" + + try: + stage_ctx = pipeline_context.get_stage_context() + except Exception: + stage_ctx = None + + in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or ""))) + start_time = time.monotonic() + time_budget = 45.0 + if in_pipeline: + try: + already_checked = bool( + pipeline_context.load_value( + "preflight.url_duplicates.checked", default=False + ) + ) + except Exception: + already_checked = False + + if already_checked: + debug("Bulk URL preflight: already checked in pipeline; skipping duplicate check") + return True + + def _load_preflight_cache() -> Dict[str, Any]: + try: + existing = pipeline_context.load_value("preflight", default=None) + except Exception: + existing = None + return existing if isinstance(existing, dict) else {} + + def _store_preflight_cache(cache: Dict[str, Any]) -> None: + try: + pipeline_context.store_value("preflight", cache) + except Exception: + pass + + def _mark_preflight_checked() -> None: + if not in_pipeline: + return + try: + pipeline_context.store_value("preflight.url_duplicates.checked", True) + except Exception: + pass + preflight_cache = _load_preflight_cache() + preflight_cache["url_duplicates_checked"] = True + url_dup_cache = preflight_cache.get("url_duplicates") + if not isinstance(url_dup_cache, dict): + url_dup_cache = {} + url_dup_cache["checked"] = True + preflight_cache["url_duplicates"] = url_dup_cache + _store_preflight_cache(preflight_cache) + + def _timed_out(reason: str) -> bool: + try: + if (time.monotonic() - start_time) >= time_budget: + debug( + f"Bulk URL preflight timed out after {time_budget:.0f}s ({reason}); continuing" + ) + _mark_preflight_checked() + return True + except Exception: + return False + return False + + if in_pipeline and auto_continue_duplicates: + try: + cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="") + cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None) + except Exception: + cached_cmd = "" + cached_decision = None + + if (not force_prompt_in_pipeline) and cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""): + _mark_preflight_checked() + if bool(cached_decision): + return True + try: + pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0) + except Exception: + pass + return False + + unique_urls: List[str] = [] + for u in urls or []: + s = str(u or "").strip() + if s and s not in unique_urls: + unique_urls.append(s) + if len(unique_urls) == 0: + return True + + try: + from SYS.metadata import normalize_urls + except Exception: + normalize_urls = None # type: ignore[assignment] + + def _httpish(value: str) -> bool: + try: + return bool(value) and (value.startswith("http://") or value.startswith("https://")) + except Exception: + return False + + def _normalize_url_for_search(value: str) -> str: + url = str(value or "").strip() + + url = url.split("#", 1)[0] + + try: + parsed = urlparse(url) + except Exception: + parsed = None + + if parsed is not None and parsed.query: + time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"} + tracking_prefixes = ("utm_",) + try: + pairs = parse_qsl(parsed.query, keep_blank_values=True) + filtered = [] + for key, val in pairs: + key_norm = str(key or "").lower() + if key_norm in time_keys: + continue + if key_norm.startswith(tracking_prefixes): + continue + filtered.append((key, val)) + if filtered: + url = urlunparse(parsed._replace(query=urlencode(filtered, doseq=True))) + else: + url = urlunparse(parsed._replace(query="")) + except Exception: + pass + + url = re.sub(r"^[a-z][a-z0-9+.-]*://", "", url, flags=re.IGNORECASE) + + url = re.sub(r"^www\.", "", url, flags=re.IGNORECASE) + + return url.lower() + + def _expand_url_variants(value: str) -> List[str]: + if not _httpish(value): + return [] + + try: + parsed = urlparse(value) + except Exception: + return [] + + if parsed.scheme.lower() not in {"http", "https"}: + return [] + + out: List[str] = [] + + def _add_variant(candidate: str) -> None: + _maybe_add(candidate) + try: + lower = str(candidate or "").lower() + except Exception: + lower = "" + if lower and lower != candidate: + _maybe_add(lower) + + try: + parsed_candidate = urlparse(candidate) + except Exception: + parsed_candidate = None + + if parsed_candidate is None: + return + + host = (parsed_candidate.hostname or "").strip().lower() + if host.startswith("www."): + host = host[4:] + if host: + netloc = host + try: + if parsed_candidate.port: + netloc = f"{netloc}:{parsed_candidate.port}" + except Exception: + pass + try: + if parsed_candidate.username or parsed_candidate.password: + userinfo = parsed_candidate.username or "" + if parsed_candidate.password: + userinfo = f"{userinfo}:{parsed_candidate.password}" + if userinfo: + netloc = f"{userinfo}@{netloc}" + except Exception: + pass + alt = urlunparse(parsed_candidate._replace(netloc=netloc)) + _maybe_add(alt) + try: + lower_alt = alt.lower() + except Exception: + lower_alt = "" + if lower_alt and lower_alt != alt: + _maybe_add(lower_alt) + + def _maybe_add(candidate: str) -> None: + if not candidate or candidate == value: + return + if candidate not in out: + out.append(candidate) + + if parsed.fragment: + _add_variant(urlunparse(parsed._replace(fragment=""))) + + time_keys = {"t", "start", "time_continue", "timestamp", "time", "begin"} + tracking_prefixes = ("utm_",) + + try: + query_pairs = parse_qsl(parsed.query, keep_blank_values=True) + except Exception: + query_pairs = [] + + if query_pairs or parsed.fragment: + filtered_pairs = [] + removed = False + for key, val in query_pairs: + key_norm = str(key or "").lower() + if key_norm in time_keys: + removed = True + continue + if key_norm.startswith(tracking_prefixes): + removed = True + continue + filtered_pairs.append((key, val)) + + if removed: + new_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else "" + _add_variant(urlunparse(parsed._replace(query=new_query, fragment=""))) + + return out + + def _dedupe_needles(raw_needles: Sequence[str]) -> List[str]: + output: List[str] = [] + seen: set[str] = set() + for candidate in (raw_needles or []): + candidate_text = str(candidate or "").strip() + if not candidate_text: + continue + key = candidate_text.lower() + if key in seen: + continue + seen.add(key) + output.append(candidate_text) + return output + + url_needles: Dict[str, List[str]] = {} + for u in unique_urls: + needles: List[str] = [] + if normalize_urls is not None: + try: + needles.extend([n for n in (normalize_urls(u) or []) if isinstance(n, str)]) + except Exception: + needles = [] + if not needles: + needles = [u] + filtered: List[str] = [] + for n in needles: + n2 = str(n or "").strip() + if not n2: + continue + if not _httpish(n2): + continue + if n2 not in filtered: + filtered.append(n2) + lowered: List[str] = [] + for n2 in filtered: + try: + lower = n2.lower() + except Exception: + lower = "" + if lower and lower != n2 and lower not in filtered and lower not in lowered: + lowered.append(lower) + normalized: List[str] = [] + for n2 in filtered: + norm = _normalize_url_for_search(n2) + if norm and norm not in normalized and norm not in filtered: + normalized.append(norm) + expanded: List[str] = [] + for n2 in filtered: + for extra in _expand_url_variants(n2): + if extra not in expanded and extra not in filtered and extra not in lowered: + expanded.append(extra) + norm_extra = _normalize_url_for_search(extra) + if ( + norm_extra + and norm_extra not in normalized + and norm_extra not in filtered + and norm_extra not in expanded + and norm_extra not in lowered + ): + normalized.append(norm_extra) + + combined = filtered + expanded + lowered + normalized + deduped = _dedupe_needles(combined) + url_needles[u] = deduped if deduped else [u] + + if in_pipeline: + preflight_cache = _load_preflight_cache() + url_dup_cache = preflight_cache.get("url_duplicates") + if not isinstance(url_dup_cache, dict): + url_dup_cache = {} + cached_urls = url_dup_cache.get("urls") + cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() + + if cached_set: + all_cached = True + for original_url, needles in url_needles.items(): + original_cached = str(original_url or "") in cached_set + needles_cached = True + if original_cached: + for needle in (needles or []): + needle_text = str(needle or "") + if not needle_text: + continue + if needle_text not in cached_set: + needles_cached = False + break + else: + needles_cached = False + + if original_cached and needles_cached: + continue + + all_cached = False + break + + if all_cached: + debug("Bulk URL preflight: cached for pipeline; skipping duplicate check") + _mark_preflight_checked() + return True + + if _timed_out("before backend scan"): + return True + + bulk_mode = len(unique_urls) > 3 + + def _build_bulk_patterns(needles_map: Dict[str, List[str]], max_per_url: int = 3, max_total: int = 240) -> List[str]: + patterns: List[str] = [] + for _original, needles in needles_map.items(): + for needle in (needles or [])[:max_per_url]: + needle_text = str(needle or "").strip() + if not needle_text: + continue + if needle_text not in patterns: + patterns.append(needle_text) + if len(patterns) >= max_total: + return patterns + return patterns + + bulk_patterns = _build_bulk_patterns(url_needles) + + def _match_normalized_url(pattern_text: str, candidate_url: str) -> bool: + pattern_norm = _normalize_url_for_search(pattern_text) + candidate_norm = _normalize_url_for_search(candidate_url) + if not pattern_norm or not candidate_norm: + return False + if pattern_norm == candidate_norm: + return True + return pattern_norm in candidate_norm + + def _extract_urls_from_hit( + hit: Any, + backend: Any, + *, + allow_backend_lookup: bool = True, + ) -> List[str]: + url_values: List[str] = [] + try: + raw_urls = get_field(hit, "known_urls") or get_field(hit, "urls") or get_field(hit, "url") + if isinstance(raw_urls, str) and raw_urls.strip(): + url_values.append(raw_urls.strip()) + elif isinstance(raw_urls, (list, tuple, set)): + for item in raw_urls: + if isinstance(item, str) and item.strip(): + url_values.append(item.strip()) + except Exception: + url_values = [] + + if url_values or not allow_backend_lookup: + return url_values + + try: + file_hash = get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or "" + except Exception: + file_hash = "" + + if file_hash: + try: + fetched = backend.get_url(str(file_hash)) + if isinstance(fetched, str) and fetched.strip(): + url_values.append(fetched.strip()) + elif isinstance(fetched, (list, tuple, set)): + for item in fetched: + if isinstance(item, str) and item.strip(): + url_values.append(item.strip()) + except Exception: + pass + + return url_values + + def _build_display_row_for_hit( + hit: Any, + backend_name: str, + original_url: str, + ) -> Dict[str, Any]: + try: + from SYS.result_table import build_display_row + extracted = build_display_row(hit, keys=["title", "store", "hash", "ext", "size"]) + except Exception: + extracted = {} + + try: + title = extracted.get("title") or get_field(hit, "title") or get_field(hit, "name") or get_field(hit, "target") or get_field(hit, "path") or "(exists)" + except Exception: + title = "(exists)" + + try: + file_hash = extracted.get("hash") or get_field(hit, "hash") or get_field(hit, "file_hash") or get_field(hit, "sha256") or "" + except Exception: + file_hash = "" + + ext = extracted.get("ext") if isinstance(extracted, dict) else "" + size_val = extracted.get("size") if isinstance(extracted, dict) else None + + return build_table_result_payload( + title=str(title), + columns=[ + ("Title", str(title)), + ("Store", str(get_field(hit, "store") or backend_name)), + ("Hash", str(file_hash or "")), + ("Ext", str(ext or "")), + ("Size", size_val), + ("URL", original_url), + ], + store=str(get_field(hit, "store") or backend_name), + hash=str(file_hash or ""), + ext=str(ext or ""), + size=size_val, + url=original_url, + ) + + def _search_backend_url_hits( + backend: Any, + backend_name: str, + original_url: str, + needles: Sequence[str], + ) -> Optional[Dict[str, Any]]: + backend_hits: List[Dict[str, Any]] = [] + + for needle in (needles or [])[:5]: + needle_stripped = str(needle or "").strip() + if not needle_stripped or not _httpish(needle_stripped): + continue + try: + query = f"url:{needle_stripped}" + backend_hits = backend.search(query, limit=1, minimal=True) or [] + if backend_hits: + return _build_display_row_for_hit(backend_hits[0], backend_name, original_url) + except Exception: + continue + + for needle in (needles or [])[:3]: + needle_text = str(needle or "").strip() + if not needle_text: + continue + search_needle = _normalize_url_for_search(needle_text) or needle_text + query = f"url:*{search_needle}*" + try: + backend_hits = backend.search(query, limit=1, minimal=True) or [] + if backend_hits: + break + except Exception: + continue + + if not backend_hits: + return None + + hit = backend_hits[0] + return _build_display_row_for_hit(hit, backend_name, original_url) + + backend_names: List[str] = [] + try: + backend_names_all = storage.list_searchable_backends() + except Exception: + backend_names_all = [] + + for backend_name in backend_names_all: + try: + backend = storage[backend_name] + except Exception: + continue + + try: + if str(backend_name).strip().lower() == "temp": + continue + except Exception: + pass + + try: + backend_location = getattr(backend, "_location", None) + if backend_location and final_output_dir: + backend_path = Path(str(backend_location)).expanduser().resolve() + temp_path = Path(str(final_output_dir)).expanduser().resolve() + if backend_path == temp_path: + continue + except Exception: + pass + + backend_names.append(backend_name) + + if not backend_names: + debug("Bulk URL preflight skipped: no searchable backends") + return True + + try: + debug_panel( + "URL preflight", + [ + ("url_count", len(unique_urls)), + ("pipeline", in_pipeline), + ("bulk_mode", bulk_mode), + ("backends", ", ".join(str(name) for name in backend_names)), + ], + border_style="yellow", + ) + except Exception: + pass + + seen_pairs: set[tuple[str, str]] = set() + matched_urls: set[str] = set() + match_rows: List[Dict[str, Any]] = [] + max_rows = 200 + + hydrus_provider = None + try: + from PluginCore.registry import get_plugin + + hydrus_provider = get_plugin("hydrusnetwork", config) + except Exception: + hydrus_provider = None + + for backend_name in backend_names: + if _timed_out("backend scan"): + return True + if len(match_rows) >= max_rows: + break + try: + backend = storage[backend_name] + except Exception: + continue + + is_hydrus_backend = False + try: + is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name))) + except Exception: + is_hydrus_backend = False + if not is_hydrus_backend: + try: + is_hydrus_backend = str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork" + except Exception: + is_hydrus_backend = False + + if is_hydrus_backend: + if not hydrus_available: + debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup") + + if _timed_out("hydrus scan"): + return True + + for original_url, needles in url_needles.items(): + if _timed_out("hydrus per-url scan"): + return True + if len(match_rows) >= max_rows: + break + if (original_url, str(backend_name)) in seen_pairs: + continue + + found_hash: Optional[str] = None + found = False + lookup_exact = getattr(backend, "find_hashes_by_url", None) + if callable(lookup_exact): + for needle in [original_url, *(needles or [])][:7]: + needle_text = str(needle or "").strip() + if not _httpish(needle_text): + continue + try: + exact_hashes = lookup_exact(needle_text) or [] + except Exception: + continue + if not isinstance(exact_hashes, list) or not exact_hashes: + continue + try: + found_hash = str(exact_hashes[0] or "").strip().lower() + except Exception: + found_hash = None + found = True + break + + if not found: + continue + + seen_pairs.add((original_url, str(backend_name))) + matched_urls.add(original_url) + display_row = build_table_result_payload( + title="(exists)", + columns=[ + ("Title", "(exists)"), + ("Store", str(backend_name)), + ("Hash", found_hash or ""), + ("URL", original_url), + ], + store=str(backend_name), + hash=found_hash or "", + url=original_url, + ) + match_rows.append(display_row) + continue + + if bulk_mode and bulk_patterns: + bulk_hits: Optional[List[Any]] = None + bulk_limit = min(2000, max(200, len(unique_urls) * 8)) + try: + bulk_hits = backend.search( + "url:*", + limit=bulk_limit, + pattern_hint=bulk_patterns, + ) or [] + except Exception: + try: + bulk_hits = backend.search("url:*", limit=bulk_limit) or [] + except Exception: + bulk_hits = None + + if bulk_hits is not None: + for hit in bulk_hits: + if _timed_out("backend bulk scan"): + return True + if len(match_rows) >= max_rows: + break + url_values = _extract_urls_from_hit(hit, backend, allow_backend_lookup=False) + if not url_values: + continue + + for original_url, needles in url_needles.items(): + if _timed_out("backend bulk scan"): + return True + if len(match_rows) >= max_rows: + break + if (original_url, str(backend_name)) in seen_pairs: + continue + + matched = False + for url_value in url_values: + for needle in (needles or []): + if _match_normalized_url(str(needle or ""), str(url_value or "")): + matched = True + break + if matched: + break + + if not matched: + continue + + seen_pairs.add((original_url, str(backend_name))) + matched_urls.add(original_url) + match_rows.append( + _build_display_row_for_hit(hit, str(backend_name), original_url) + ) + continue + + for original_url, needles in url_needles.items(): + if _timed_out("backend per-url scan"): + return True + if len(match_rows) >= max_rows: + break + if (original_url, str(backend_name)) in seen_pairs: + continue + + display_row = _search_backend_url_hits(backend, str(backend_name), original_url, needles) + if not display_row: + continue + + seen_pairs.add((original_url, str(backend_name))) + matched_urls.add(original_url) + match_rows.append(display_row) + + if not match_rows: + if in_pipeline: + preflight_cache = _load_preflight_cache() + url_dup_cache = preflight_cache.get("url_duplicates") + if not isinstance(url_dup_cache, dict): + url_dup_cache = {} + + cached_urls = url_dup_cache.get("urls") + cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() + + for original_url, needles in url_needles.items(): + cached_set.add(original_url) + for needle in needles or []: + cached_set.add(str(needle)) + + url_dup_cache["urls"] = sorted(cached_set) + preflight_cache["url_duplicates"] = url_dup_cache + _store_preflight_cache(preflight_cache) + _mark_preflight_checked() + return True + + table = Table(f"URL already exists ({len(matched_urls)} url(s))", max_columns=10) + table._interactive(True) + try: + table._perseverance(True) + except Exception: + pass + + for row in match_rows: + table.add_result(row) + + try: + pipeline_context.set_last_result_table_overlay(table, match_rows) + except Exception: + pass + + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): + try: + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + except Exception: + cm = nullcontext() + + auto_confirm_reason: Optional[str] = None + if in_pipeline and stage_ctx is not None: + try: + total_stages = int(getattr(stage_ctx, "total_stages", 0)) + except Exception: + total_stages = 0 + try: + is_last_stage = bool(getattr(stage_ctx, "is_last_stage", False)) + except Exception: + is_last_stage = False + if total_stages > 1 and not is_last_stage and not force_prompt_in_pipeline: + auto_confirm_reason = "pipeline stage (pre-last)" + if auto_confirm_reason is None: + try: + stdin_interactive = bool(sys.stdin and sys.stdin.isatty()) + except Exception: + stdin_interactive = False + if not stdin_interactive: + auto_confirm_reason = "non-interactive stdin" + + answered_yes = True + auto_declined = False + with cm: + get_stderr_console().print(table) + setattr(table, "_rendered_by_cmdlet", True) + if auto_confirm_reason is None: + answered_yes = bool(Confirm.ask("Continue?", default=False, console=get_stderr_console())) + else: + answered_yes = bool(auto_continue_duplicates) + auto_declined = not answered_yes + if answered_yes: + debug( + f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input." + ) + try: + log( + f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...", + file=sys.stderr, + ) + except Exception: + pass + else: + debug( + f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input." + ) + try: + log( + f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.", + file=sys.stderr, + ) + except Exception: + pass + + if in_pipeline and auto_continue_duplicates: + try: + existing = pipeline_context.load_value("preflight", default=None) + except Exception: + existing = None + preflight_cache: Dict[str, Any] = existing if isinstance(existing, dict) else {} + url_dup_cache = preflight_cache.get("url_duplicates") + if not isinstance(url_dup_cache, dict): + url_dup_cache = {} + url_dup_cache["command"] = str(current_cmd_text or "") + url_dup_cache["continue"] = bool(answered_yes) + cached_urls = url_dup_cache.get("urls") + cached_set = {str(u) for u in cached_urls} if isinstance(cached_urls, list) else set() + for original_url, needles in url_needles.items(): + cached_set.add(original_url) + for needle in needles or []: + cached_set.add(str(needle)) + url_dup_cache["urls"] = sorted(cached_set) + preflight_cache["url_duplicates"] = url_dup_cache + try: + pipeline_context.store_value("preflight", preflight_cache) + except Exception: + pass + + if not answered_yes: + if in_pipeline and not auto_declined: + try: + pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0) + except Exception: + pass + _mark_preflight_checked() + return False + _mark_preflight_checked() + return True diff --git a/cmdlet/file/add.py b/cmdlet/file/add.py index f7a1cf1..93570ea 100644 --- a/cmdlet/file/add.py +++ b/cmdlet/file/add.py @@ -1,3270 +1,63 @@ -from __future__ import annotations +"""add-file cmdlet — re-export module. -from typing import Any, Dict, Optional, Sequence, Tuple, List -from pathlib import Path -import sys -import shutil -import tempfile -import re -from urllib.parse import urlparse +This module re-exports the public API from the split sub-modules to maintain +backward compatibility with existing imports. +""" -from SYS import models -from SYS import pipeline as ctx -from SYS.logger import log, debug, debug_panel -from SYS.payload_builders import build_table_result_payload -from SYS.pipeline_progress import PipelineProgress -from SYS.result_publication import overlay_existing_result_table, publish_result_table -from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel -from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS -from PluginCore.backend_registry import BackendRegistry -from API.HTTP import download_direct_file -from .. import _shared as sh +from .add_core import ( + Add_File, + CMDLET, + _CommandDependencies, + SUPPORTED_MEDIA_EXTENSIONS, + _REMOTE_URL_PREFIXES, + _SCREENSHOT_TIME_SUFFIX_RE, +) +from .add_tagging import _maybe_apply_florencevision_tags -Cmdlet = sh.Cmdlet -CmdletArg = sh.CmdletArg -parse_cmdlet_args = sh.parse_cmdlet_args -SharedArgs = sh.SharedArgs -extract_tag_from_result = sh.extract_tag_from_result -extract_title_from_result = sh.extract_title_from_result -extract_url_from_result = sh.extract_url_from_result -merge_sequences = sh.merge_sequences -extract_relationships = sh.extract_relationships -extract_duration = sh.extract_duration -coerce_to_pipe_object = sh.coerce_to_pipe_object -collapse_namespace_tags = sh.collapse_namespace_tags -resolve_target_dir = sh.resolve_target_dir -resolve_media_kind_by_extension = sh.resolve_media_kind_by_extension -coerce_to_path = sh.coerce_to_path -build_pipeline_preview = sh.build_pipeline_preview -get_field = sh.get_field - -from SYS.utils import sha256_file, unique_path, sanitize_filename - -# Canonical supported filetypes for all stores/cmdlets -SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS -_SCREENSHOT_TIME_SUFFIX_RE = re.compile( - r"^(?P.+?)_(?P<label>(?:\d+h)?(?:\d+m)?\d+s)$", - flags=re.IGNORECASE, +# Re-export sub-module functions for direct import if needed +from .add_validation import ( + _resolve_source, + _validate_source, + _scan_directory_for_files, + _is_probable_url, + _resolve_backend_by_name, + _download_piped_source, + _maybe_download_plugin_result, + _build_provider_filename, + _maybe_download_backend_file, + _download_remote_backend_url, ) - -class _CommandDependencies: - """Command-scope cache for the backend registry and plugin instances.""" - - def __init__(self, config: Dict[str, Any]) -> None: - self.config = config - self._backend_registry: Optional[BackendRegistry] = None - self._plugins: Dict[str, Any] = {} - - def get_backend_registry(self) -> Optional[BackendRegistry]: - """Lazily initialize and return the command-scope backend registry.""" - if self._backend_registry is None: - try: - self._backend_registry = BackendRegistry(self.config) - except Exception: - self._backend_registry = None - return self._backend_registry - - def get_plugin(self, name: str) -> Optional[Any]: - """Cached plugin lookup by name.""" - from PluginCore.registry import get_plugin - - norm_name = str(name or "").strip().lower() - if not norm_name: - return None - if norm_name in self._plugins: - return self._plugins[norm_name] - - plugin = get_plugin(norm_name, self.config) - self._plugins[norm_name] = plugin - return plugin - - def get_plugin_for_cmdlet(self, name: str, cmdlet_name: str) -> Optional[Any]: - """Cached plugin lookup with explicit cmdlet support check.""" - from PluginCore.registry import get_plugin_for_cmdlet - - norm_name = str(name or "").strip().lower() - if not norm_name: - return None - - cache_key = f"{norm_name}#{cmdlet_name}" - if cache_key in self._plugins: - return self._plugins[cache_key] - - plugin = get_plugin_for_cmdlet(norm_name, cmdlet_name, self.config) - self._plugins[cache_key] = plugin - return plugin - -_REMOTE_URL_PREFIXES: tuple[str, ...] = ( - "http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:", "tidal:", "hydrus:", +from .add_tagging import ( + _prepare_metadata, + _resolve_file_hash, + _load_sidecar_bundle, + _get_url, + _get_relationships, + _get_duration, + _get_note_text, + _parse_relationship_tag_king_alts, + _parse_relationships_king_alts, + _normalize_hash_candidate, + _resolve_media_kind, ) - -def _maybe_apply_florencevision_tags( - media_path: Path, - tags: List[str], - config: Dict[str, Any], - pipe_obj: Optional[models.PipeObject] = None, -) -> List[str]: - """Optionally auto-tag images using the FlorenceVision plugin helper. - - Controlled via config: - [plugin=florencevision] - enabled=true - strict=false - - If strict=false (default), failures log a warning and return the original tags. - If strict=true, failures raise to abort the ingest. - """ - strict = False - try: - plugin_block = (config or {}).get("plugin") - fv_block = plugin_block.get("florencevision") if isinstance(plugin_block, dict) else None - enabled = False - if isinstance(fv_block, dict): - enabled = bool(fv_block.get("enabled")) - strict = bool(fv_block.get("strict")) - if not enabled: - return tags - - from plugins.florencevision import FlorenceVisionTool - - # Special-case: if this file was produced by the `screen-shot` cmdlet, - # OCR is more useful than caption/detection for tagging screenshots. - cfg_for_tool: Dict[str, Any] = config - try: - action = str(getattr(pipe_obj, "action", "") or "") if pipe_obj is not None else "" - cmdlet_name = "" - if action.lower().startswith("cmdlet:"): - cmdlet_name = action.split(":", 1)[1].strip().lower() - if cmdlet_name in {"screen-shot", "screen_shot", "screenshot"}: - plugin_block2 = dict((config or {}).get("plugin") or {}) - fv_block2 = dict(plugin_block2.get("florencevision") or {}) - fv_block2["task"] = "ocr" - plugin_block2["florencevision"] = fv_block2 - cfg_for_tool = dict(config or {}) - cfg_for_tool["plugin"] = plugin_block2 - except Exception: - cfg_for_tool = config - - fv = FlorenceVisionTool(cfg_for_tool) - if not fv.enabled() or not fv.applicable_path(media_path): - return tags - - auto_tags = fv.tags_for_file(media_path) - - # Capture caption (if any) into PipeObject notes for downstream persistence. - try: - caption_text = getattr(fv, "last_caption", None) - if caption_text and pipe_obj is not None: - if not isinstance(pipe_obj.extra, dict): - pipe_obj.extra = {} - notes = pipe_obj.extra.get("notes") - if not isinstance(notes, dict): - notes = {} - notes.setdefault("caption", caption_text) - pipe_obj.extra["notes"] = notes - except Exception: - pass - - if not auto_tags: - return tags - - merged = merge_sequences(tags or [], auto_tags, case_sensitive=False) - debug(f"[add-file] FlorenceVision added {len(auto_tags)} tag(s)") - return merged - except Exception as exc: - # Decide strictness from config if we couldn't read it above. - strict2 = False - try: - tool_block = (config or {}).get("tool") - fv_block = tool_block.get("florencevision") if isinstance(tool_block, dict) else None - strict2 = bool(fv_block.get("strict")) if isinstance(fv_block, dict) else False - except Exception: - strict2 = False - - if strict or strict2: - raise - log(f"[add-file] Warning: FlorenceVision tagging failed: {exc}", file=sys.stderr) - return tags - - -class Add_File(Cmdlet): - """Add file into the DB""" - - def __init__(self) -> None: - """Initialize add-file cmdlet.""" - super().__init__( - name="add-file", - summary= - "Ingest a local media file to a configured store or plugin destination.", - usage= - "add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-folder <name>] [-delete]", - arg=[ - CmdletArg( - name="source", - type="string", - required=False, - description="Local file or directory path to ingest or scan.", - ), - SharedArgs.INSTANCE, - SharedArgs.URL, - SharedArgs.PLUGIN, - CmdletArg( - name="delete", - type="flag", - required=False, - description="Delete file after successful upload", - alias="del", - ), - CmdletArg( - name="folder", - description="Folder name override for local plugin exports.", - alias="folder-name", - ), - ], - detail=[ - "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", - "- Store options (use -instance without -plugin):", - " hydrus: Upload to Hydrus database with metadata tagging", - "- Plugin options (use -plugin):", - " local: Copy file to a configured local destination or direct path via -instance", - " local folder exports: use -folder <name> (or -folder-name); piped AllDebrid downloads default to their magnet folder", - " 0x0: Upload to 0x0.st for temporary hosting", - " file.io: Upload to file.io for temporary hosting", - " internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)", - "- Use a positional source path with -instance and -plugin to target a named provider config: add-file C:\\Media\\file.pdf -plugin ftp -instance archive", - ], - examples=[ - 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial', - '@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads', - 'add-file C:\\Media\\report.pdf -plugin ftp -instance archive', - ], - exec=self.run, - ) - self.register() - - @staticmethod - def _uses_legacy_path_flag(args: Sequence[str]) -> bool: - for token in args or []: - lowered = str(token or "").strip().lower() - if lowered in {"-path", "--path", "-p"}: - return True - return False - - @staticmethod - def _legacy_path_flag_message() -> str: - return ( - "add-file no longer supports -path. Pass the source file or directory as a positional argument, " - "and use -plugin local -instance <name|path> for local export." - ) - - def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: - """Main execution entry point.""" - if Add_File._uses_legacy_path_flag(args): - log(Add_File._legacy_path_flag_message(), file=sys.stderr) - return 1 - - parsed = parse_cmdlet_args(args, self) - progress = PipelineProgress(ctx) - - # Initialize command-scope dependency context (caches Store/plugins) - deps = _CommandDependencies(config) - storage_registry: Optional[BackendRegistry] = None - - source_arg = parsed.get("source") - location = parsed.get("instance") - plugin_instance = parsed.get("instance") - source_url_arg = parsed.get("url") - plugin_name = parsed.get("plugin") - delete_after = parsed.get("delete", False) - folder_name = parsed.get("folder") - local_export_destination: Optional[str] = None - if plugin_name and not plugin_instance and location: - plugin_instance = location - - stage_ctx = ctx.get_stage_context() - is_last_stage = (stage_ctx - is None) or bool(getattr(stage_ctx, - "is_last_stage", - False)) - has_downstream_stage = bool(stage_ctx is not None and not is_last_stage) - - # Directory-mode selector: - # - Terminal use: `add-file <DIR> -instance X` shows a selectable table. - # - Pipelined use: `add-file <DIR> -instance X | ...` processes the full batch - # immediately so downstream stages receive the uploaded items. - # - Selection replay: `@N` re-runs add-file with `file1,file2,...` as the source token. - dir_scan_mode = False - dir_scan_results: Optional[List[Dict[str, Any]]] = None - explicit_source_list_results: Optional[List[Dict[str, Any]]] = None - - if source_arg and location and not plugin_name: - # Support comma-separated source lists: "file1,file2,file3" - # This is the mechanism used by @N expansion for directory tables. - try: - source_text = str(source_arg) - except Exception: - source_text = "" - - if "," in source_text: - parts = [p.strip().strip('"') for p in source_text.split(",")] - parts = [p for p in parts if p] - - batch: List[Dict[str, Any]] = [] - for p in parts: - try: - file_path = Path(p) - except Exception: - continue - if not file_path.exists() or not file_path.is_file(): - continue - ext = file_path.suffix.lower() - if ext not in SUPPORTED_MEDIA_EXTENSIONS: - continue - try: - hv = sha256_file(file_path) - except Exception: - continue - try: - size = file_path.stat().st_size - except Exception: - size = 0 - batch.append( - { - "path": file_path, - "name": file_path.name, - "hash": hv, - "size": size, - "ext": ext, - } - ) - - if batch: - explicit_source_list_results = batch - # Clear source_arg so add-file doesn't treat it as a single path. - source_arg = None - else: - # Directory scan (selector table, no ingest yet) - try: - candidate_dir = Path(str(source_arg)) - if candidate_dir.exists() and candidate_dir.is_dir(): - dir_scan_mode = True - debug( - f"[add-file] Scanning directory for batch add: {candidate_dir}" - ) - dir_scan_results = Add_File._scan_directory_for_files( - candidate_dir - ) - if dir_scan_results: - debug( - f"[add-file] Found {len(dir_scan_results)} supported files in directory" - ) - # Clear source_arg so it doesn't trigger single-item mode. - source_arg = None - except Exception as exc: - debug(f"[add-file] Directory scan failed: {exc}") - - if result is None and not source_arg and not explicit_source_list_results and not dir_scan_results: - try: - if ctx.get_stage_context() is not None: - return 0 - except Exception: - pass - - # Determine if -instance targets a registered backend (vs a filesystem export path). - is_storage_backend_location = False - if location and not plugin_name: - try: - backend_registry_for_lookup = storage_registry or deps.get_backend_registry() - storage_registry = backend_registry_for_lookup - is_storage_backend_location = Add_File._resolve_backend_by_name(backend_registry_for_lookup, str(location)) is not None - except Exception: - is_storage_backend_location = False - - if location and not plugin_name and not is_storage_backend_location: - resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( - location, - config, - deps=deps, - require_explicit=True, - ) - if resolved_local_path: - plugin_name = "local" - plugin_instance = resolved_local_instance or str(location) - location = None - local_export_destination = resolved_local_path - else: - log( - f"Storage backend '{location}' not found. Use -plugin local -instance <name|path> for local export or configure that store backend.", - file=sys.stderr, - ) - return 1 - - normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) - if normalized_plugin_name == "local": - resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( - plugin_instance or location, - config, - deps=deps, - require_explicit=bool(plugin_instance or location), - ) - if not resolved_local_path: - requested_local = str(plugin_instance or location or "").strip() or "<default>" - log( - f"Local destination '{requested_local}' is not configured. Use -plugin local -instance <name|path>.", - file=sys.stderr, - ) - return 1 - plugin_name = "local" - plugin_instance = resolved_local_instance or str(plugin_instance or location or "").strip() or None - location = None - local_export_destination = resolved_local_path - - plugin_storage_backend = None - if plugin_name: - plugin_storage_backend = Add_File._resolve_plugin_storage_backend( - plugin_name, - plugin_instance, - config, - store_instance=storage_registry, - deps=deps, - ) - if plugin_storage_backend and storage_registry is None: - storage_registry = deps.get_backend_registry() - - effective_storage_backend_name = plugin_storage_backend or ( - str(location) if location and is_storage_backend_location else None - ) - - # Decide which items to process. - # - If directory scan was performed, use those results - # - If user provided a positional source path, treat this invocation as single-item. - # - Otherwise, if piped input is a list, ingest each item. - if explicit_source_list_results: - items_to_process = explicit_source_list_results - debug(f"[add-file] Using {len(items_to_process)} files from source list") - elif dir_scan_results: - items_to_process = dir_scan_results - debug(f"[add-file] Using {len(items_to_process)} files from directory scan") - elif source_arg: - items_to_process: List[Any] = [result] - elif isinstance(result, list) and result: - items_to_process = list(result) - else: - items_to_process = [result] - - total_items = len(items_to_process) if isinstance(items_to_process, list) else 0 - processed_items = 0 - try: - ui, _ = progress.ui_and_pipe_index() - if ui is not None and total_items: - preview_items = ( - list(items_to_process) - if isinstance(items_to_process, list) else [items_to_process] - ) - progress.begin_pipe( - total_items=total_items, - items_preview=preview_items, - ) - except Exception: - pass - try: - if total_items: - progress.set_percent(0) - except Exception: - pass - - # Minimal step-based progress for single-item runs. - # Many add-file flows don't emit intermediate items, so without steps the pipe can look "stuck". - use_steps = False - steps_started = False - try: - ui, _ = progress.ui_and_pipe_index() - use_steps = (ui is not None) and (len(items_to_process) == 1) - if use_steps: - progress.begin_steps(5) - steps_started = True - except Exception: - use_steps = False - - # add-file is ingestion-only: it does not download URLs here. - - should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage) - if dir_scan_mode and has_downstream_stage: - debug( - "[add-file] Continuing with directory batch ingest because downstream stages exist" - ) - - # If this invocation was terminal directory selector mode, show a selectable table and stop. - # The user then runs @N (optionally piped), which replays add-file with selected source paths. - if should_present_directory_selector: - try: - from SYS.result_table import Table - from pathlib import Path as _Path - - base_args: List[str] = [] - if plugin_name: - base_args.extend(["-plugin", str(plugin_name)]) - if location: - base_args.extend(["-instance", str(location)]) - if source_url_arg: - base_args.extend(["-url", str(source_url_arg)]) - if bool(delete_after): - base_args.append("-delete") - - table = Table(title="Files in Directory", preserve_order=True) - table.set_table("add-file.directory") - table.set_source_command("add-file", base_args) - - rows: List[Dict[str, Any]] = [] - for file_info in dir_scan_results or []: - p = file_info.get("path") - hp = str(file_info.get("hash") or "") - name = str(file_info.get("name") or "unknown") - try: - clean_title = _Path(name).stem - except Exception: - clean_title = name - ext = str(file_info.get("ext") or "").lstrip(".") - size = file_info.get("size", 0) - - row_item = build_table_result_payload( - title=clean_title, - columns=[ - ("Title", clean_title), - ("Hash", hp), - ("Size", size), - ("Ext", ext), - ], - selection_args=[str(p) if p is not None else ""], - path=str(p) if p is not None else "", - hash=hp, - ) - rows.append(row_item) - table.add_result(row_item) - - ctx.set_current_stage_table(table) - ctx.set_last_result_table( - table, - rows, - subject={ - "table": "add-file.directory" - } - ) - log(f"✓ Found {len(rows)} files. Select with @N (e.g., @1 or @1-3).") - return 0 - except Exception as exc: - debug( - f"[add-file] Failed to display directory scan result table: {exc}" - ) - - collected_payloads: List[Dict[str, Any]] = [] - pending_relationship_pairs: Dict[str, - set[tuple[str, - str]]] = {} - pending_url_associations: Dict[str, - List[tuple[str, - List[str]]]] = {} - pending_tag_associations: Dict[str, - List[tuple[str, - List[str]]]] = {} - successes = 0 - failures = 0 - - # When add-file -instance is the last stage, always show a final search-file table. - # This is especially important for multi-item ingests (e.g., multi-clip downloads) - # so the user always gets a selectable ResultTable. - live_progress = None - try: - live_progress = ctx.get_live_progress() - except Exception: - live_progress = None - - want_final_search_file = ( - bool(is_last_stage) - and bool(effective_storage_backend_name) - and bool(live_progress) - ) - auto_search_file_after_add = False - - # When ingesting multiple items into a backend store, defer URL association and - # apply it once at the end (bulk) to avoid per-item URL API calls. - defer_url_association = ( - bool(effective_storage_backend_name) - and len(items_to_process) > 1 - ) - - for idx, item in enumerate(items_to_process, 1): - pipe_obj = coerce_to_pipe_object(item, source_arg) - - if source_url_arg: - try: - from SYS.metadata import normalize_urls - - cli_urls = [u.strip() for u in str(source_url_arg).split(",") if u and u.strip()] - merged_urls: List[str] = [] - - if isinstance(getattr(pipe_obj, "extra", None), dict): - existing_url = pipe_obj.extra.get("url") - if isinstance(existing_url, list): - merged_urls.extend(str(u) for u in existing_url if u) - elif isinstance(existing_url, str) and existing_url.strip(): - merged_urls.append(existing_url.strip()) - else: - pipe_obj.extra = {} - - merged_urls = sh.merge_urls(merged_urls, cli_urls) - if merged_urls: - pipe_obj.extra["url"] = merged_urls - except Exception: - pass - - try: - label = pipe_obj.title - if not label and pipe_obj.path: - try: - label = Path(str(pipe_obj.path)).name - except Exception: - label = pipe_obj.path - if not label: - label = "file" - if total_items: - pending_pct = int(round(((idx - 1) / max(1, total_items)) * 100)) - progress.set_percent(pending_pct) - progress.set_status(f"adding {idx}/{total_items}: {label}") - except Exception: - pass - - temp_dir_to_cleanup: Optional[Path] = None - delete_after_item = delete_after - try: - if use_steps and steps_started: - progress.step("resolving source") - - export_destination = ( - Path(local_export_destination) - if local_export_destination - else Path(location) - if location and not is_storage_backend_location - else None - ) - media_path, file_hash, temp_dir_to_cleanup = self._resolve_source( - item, - source_arg, - pipe_obj, - config, - export_destination=export_destination, - store_instance=storage_registry, - deps=deps, - ) - if not media_path and plugin_name: - media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source( - pipe_obj, config, storage_registry, deps=deps - ) - if not media_path: - failures += 1 - continue - - # Update pipe_obj with resolved path - pipe_obj.path = str(media_path) - - # Local/plugin exports can accept any file type. - # Storage backends stay restricted to SUPPORTED_MEDIA_EXTENSIONS. - allow_all_files = not bool(effective_storage_backend_name) - if not self._validate_source(media_path, allow_all_extensions=allow_all_files): - failures += 1 - continue - - if use_steps and steps_started: - if not file_hash: - progress.step("hashing file") - progress.step("ingesting file") - - if plugin_name: - if effective_storage_backend_name: - code = self._handle_storage_backend( - item, - media_path, - effective_storage_backend_name, - pipe_obj, - config, - delete_after_item, - collect_payloads=collected_payloads, - collect_relationship_pairs=pending_relationship_pairs, - defer_url_association=defer_url_association, - pending_url_associations=pending_url_associations, - defer_tag_association=defer_url_association, - pending_tag_associations=pending_tag_associations, - suppress_last_stage_overlay=want_final_search_file, - auto_search_file=auto_search_file_after_add, - store_instance=storage_registry, - ) - else: - code = self._handle_plugin_upload( - media_path, - plugin_name, - plugin_instance, - pipe_obj, - config, - delete_after_item, - folder_name=folder_name, - ) - if code == 0: - successes += 1 - else: - failures += 1 - continue - - if location: - try: - backend_registry = storage_registry or deps.get_backend_registry() - resolved_backend = Add_File._resolve_backend_by_name(backend_registry, str(location)) - if resolved_backend is not None: - code = self._handle_storage_backend( - item, - media_path, - location, - pipe_obj, - config, - delete_after_item, - collect_payloads=collected_payloads, - collect_relationship_pairs=pending_relationship_pairs, - defer_url_association=defer_url_association, - pending_url_associations=pending_url_associations, - defer_tag_association=defer_url_association, - pending_tag_associations=pending_tag_associations, - suppress_last_stage_overlay=want_final_search_file, - auto_search_file=auto_search_file_after_add, - store_instance=storage_registry, - ) - else: - log(f"Invalid storage backend: {location}", file=sys.stderr) - code = 1 - except Exception as exc: - debug(f"[add-file] ERROR: Failed to resolve location: {exc}") - log(f"Invalid location: {location}", file=sys.stderr) - failures += 1 - continue - - if code == 0: - successes += 1 - else: - failures += 1 - continue - - log("No destination specified", file=sys.stderr) - failures += 1 - finally: - if temp_dir_to_cleanup is not None: - try: - shutil.rmtree(temp_dir_to_cleanup, ignore_errors=True) - except Exception: - pass - processed_items += 1 - try: - pct = int(round((processed_items / max(1, total_items)) * 100)) - progress.set_percent(pct) - if processed_items >= total_items: - progress.clear_status() - except Exception: - pass - - # Apply deferred url associations (bulk) before showing the final store table. - if pending_url_associations: - try: - Add_File._apply_pending_url_associations( - pending_url_associations, - config, - store_instance=storage_registry - ) - except Exception: - pass - - # Apply deferred tag associations (bulk) if collected - if pending_tag_associations: - try: - Add_File._apply_pending_tag_associations( - pending_tag_associations, - config, - store_instance=storage_registry - ) - except Exception: - pass - - # Always end add-file -instance (when last stage) by showing item detail panels. - # Legacy search-file refresh is no longer used for final display. - if want_final_search_file and collected_payloads: - try: - from SYS.rich_display import render_item_details_panel - from SYS.result_table import Table - - # Stop the live pipeline progress UI before rendering the details panels. - # This prevents the progress display from lingering on screen. - Add_File._stop_live_progress_for_terminal_render() - - subject = collected_payloads[0] if len(collected_payloads) == 1 else collected_payloads - # Use helper to display items and make them @-selectable - from .._shared import display_and_persist_items - display_and_persist_items(collected_payloads, title="Result", subject=subject) - - try: - ctx.set_last_result_items_only(list(collected_payloads)) - except Exception: - pass - except Exception: - pass - - # Persist relationships into backend DB/API. - if pending_relationship_pairs: - try: - Add_File._apply_pending_relationships( - pending_relationship_pairs, - config, - store_instance=storage_registry, - deps=deps - ) - except Exception: - pass - - if use_steps and steps_started: - progress.step("finalized") - # Clear the status so it doesn't linger in the UI - progress.clear_status() - - if successes > 0: - return 0 - return 1 - - @staticmethod - def _try_emit_search_file_by_hashes( - *, - instance: str, - hash_values: List[str], - config: Dict[str, - Any], - store_instance: Optional[BackendRegistry] = None, - ) -> Optional[List[Any]]: - """Run search-file for a list of hashes and promote the table to a display overlay. - - Returns the emitted search-file payload items on success, else None. - """ - hashes = [h for h in (hash_values or []) if isinstance(h, str) and len(h) == 64] - if not instance or not hashes: - return None - - try: - from cmdlet.file.search import CMDLET as search_file_cmdlet - - query = "hash:" + ",".join(hashes) - args = ["-instance", str(instance), "-internal-refresh", query] - debug(f'[add-file] Refresh: search-file -instance {instance} "{query}"') - - # Run search-file under a temporary stage context so its ctx.emit() calls - # don't interfere with the outer add-file pipeline stage. - prev_ctx = ctx.get_stage_context() - temp_ctx = ctx.PipelineStageContext( - stage_index=0, - total_stages=1, - pipe_index=0, - worker_id=getattr(prev_ctx, - "worker_id", - None), - ) - ctx.set_stage_context(temp_ctx) - try: - code = search_file_cmdlet.run(None, args, config) - emitted_items = list(getattr(temp_ctx, "emits", []) or []) - finally: - ctx.set_stage_context(prev_ctx) - - if code != 0: - return None - - # Promote the search-file result to a display overlay so the CLI prints it - # for action commands like add-file. - stage_ctx = ctx.get_stage_context() - is_last = (stage_ctx - is None) or bool(getattr(stage_ctx, - "is_last_stage", - False)) - if is_last: - try: - table = ctx.get_last_result_table() - items = ctx.get_last_result_items() - if table is not None and items: - # If we have a single item refresh, render it as a panel immediately - # and suppress the table output from the CLI runner. - if len(items) == 1: - try: - from SYS.rich_display import render_item_details_panel - render_item_details_panel(items[0]) - setattr(table, "_rendered_by_cmdlet", True) - except Exception as exc: - debug(f"[add-file] Item details render failed: {exc}") - - publish_result_table( - ctx, - table, - items, - subject={ - "store": instance, - "hash": hashes - }, - overlay=True, - ) - except Exception: - pass - - return emitted_items - except Exception as exc: - debug( - f"[add-file] Failed to run search-file after add-file: {type(exc).__name__}: {exc}" - ) - return None - - @staticmethod - def _parse_relationship_tag_king_alts( - tag_value: str - ) -> tuple[Optional[str], - List[str]]: - """Parse a relationship tag into (king_hash, alt_hashes). - - Supported formats: - - New: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH> - - Old: relationship: hash(king)<KING_HASH>,hash(alt)<ALT_HASH>... - relationship: hash(king)KING,hash(alt)ALT - - For the local DB we treat the first hash listed as the king. - """ - if not isinstance(tag_value, str): - return None, [] - - raw = tag_value.strip() - if not raw: - return None, [] - - # Normalize input: ensure we only look at the RHS after "relationship:" - rhs = raw - if ":" in raw: - prefix, rest = raw.split(":", 1) - if prefix.strip().lower() == "relationship": - rhs = rest.strip() - - # Old typed format: hash(type)HEX - typed = re.findall(r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?", rhs) - if typed: - king: Optional[str] = None - alts: List[str] = [] - for rel_type, h in typed: - h_norm = str(h).strip().lower() - if rel_type.strip().lower() == "king": - king = h_norm - elif rel_type.strip().lower() in {"alt", - "related"}: - alts.append(h_norm) - # If the tag omitted king but had hashes, fall back to first hash. - if not king: - all_hashes = [str(h).strip().lower() for _, h in typed] - king = all_hashes[0] if all_hashes else None - alts = [h for h in all_hashes[1:] if h] - # Dedupe alts while preserving order - seen: set[str] = set() - alts = [ - h for h in alts - if h and len(h) == 64 and not (h in seen or seen.add(h)) - ] - if king and len(king) == 64: - return king, [h for h in alts if h != king] - return None, [] - - # New format: a simple list of hashes, first is king. - hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", rhs) - hashes = [h.strip().lower() for h in hashes if isinstance(h, str)] - if not hashes: - return None, [] - king = hashes[0] - alts = hashes[1:] - seen2: set[str] = set() - alts = [ - h for h in alts if h and len(h) == 64 and not (h in seen2 or seen2.add(h)) - ] - return king, [h for h in alts if h != king] - - @staticmethod - def _parse_relationships_king_alts( - relationships: Dict[str, - Any], - ) -> tuple[Optional[str], - List[str]]: - """Parse a PipeObject.relationships dict into (king_hash, alt_hashes). - - Supported shapes: - - {"king": [KING], "alt": [ALT1, ALT2]} - - {"king": KING, "alt": ALT} (strings) - - Also treats "related" hashes as alts for persistence purposes. - """ - if not isinstance(relationships, dict) or not relationships: - return None, [] - - def _first_hash(val: Any) -> Optional[str]: - if isinstance(val, str): - h = val.strip().lower() - return h if len(h) == 64 else None - if isinstance(val, list): - for item in val: - if isinstance(item, str): - h = item.strip().lower() - if len(h) == 64: - return h - return None - - def _many_hashes(val: Any) -> List[str]: - out: List[str] = [] - if isinstance(val, str): - h = val.strip().lower() - if len(h) == 64: - out.append(h) - elif isinstance(val, list): - for item in val: - if isinstance(item, str): - h = item.strip().lower() - if len(h) == 64: - out.append(h) - return out - - king = _first_hash(relationships.get("king")) - if not king: - return None, [] - - alts = _many_hashes(relationships.get("alt")) - alts.extend(_many_hashes(relationships.get("related"))) - - seen: set[str] = set() - alts = [h for h in alts if h and h != king and not (h in seen or seen.add(h))] - return king, alts - - @staticmethod - def _apply_pending_relationships( - pending: Dict[str, - set[tuple[str, - str]]], - config: Dict[str, - Any], - store_instance: Optional[BackendRegistry] = None, - deps: Optional[_CommandDependencies] = None, - ) -> None: - """Persist relationships to backends that support relationships. - - This delegates to an optional backend method: `set_relationship(alt, king, kind)`. - """ - if not pending: - return - - if deps is None: - deps = _CommandDependencies(config) - - try: - backend_registry = store_instance if store_instance is not None else deps.get_backend_registry() - except Exception: - return - - for backend_name, pairs in pending.items(): - if not pairs: - continue - - try: - backend = backend_registry[str(backend_name)] - except Exception: - continue - - if not bool(getattr(backend, "supports_relationship_association", False)): - continue - - setter = getattr(backend, "set_relationship", None) - if not callable(setter): - continue - - processed_pairs: set[tuple[str, str]] = set() - for alt_hash, king_hash in sorted(pairs): - if not alt_hash or not king_hash or alt_hash == king_hash: - continue - if (alt_hash, king_hash) in processed_pairs: - continue - alt_norm = str(alt_hash).strip().lower() - king_norm = str(king_hash).strip().lower() - if len(alt_norm) != 64 or len(king_norm) != 64: - continue - try: - setter(alt_norm, king_norm, "alt") - processed_pairs.add((alt_hash, king_hash)) - except Exception: - continue - - @staticmethod - def _maybe_download_backend_file( - backend: Any, - file_hash: str, - pipe_obj: models.PipeObject, - *, - output_dir: Optional[Path] = None, - ) -> Tuple[Optional[Path], Optional[Path]]: - """Best-effort fetch of a backend file when get_file returns a URL. - - Returns (downloaded_path, temp_dir_to_cleanup). - """ - - downloader = getattr(backend, "download_to_temp", None) - if not callable(downloader): - return None, None - - tmp_dir: Optional[Path] = None - try: - # Extract suffix from pipe_obj path to avoid .tmp rejections - suffix = None - if pipe_obj.path: - try: - suffix = Path(pipe_obj.path).suffix - except Exception: - pass - - # Extract suffix from metadata if available (fallback) - if not suffix: - metadata = getattr(pipe_obj, "metadata", {}) - if isinstance(metadata, dict): - suffix = metadata.get("ext") - - download_root = output_dir - if download_root is None: - tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) - download_root = tmp_dir - if download_root is None: - return None, None - - # Introspect downloader to pass supported args. - import inspect - - sig = inspect.signature(downloader) - kwargs = {"temp_root": download_root} - if "suffix" in sig.parameters: - kwargs["suffix"] = suffix - - pipeline_progress = PipelineProgress(ctx) - transfer_label = "peer transfer" - try: - transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label - except Exception: - transfer_label = "peer transfer" - if "pipeline_progress" in sig.parameters: - kwargs["pipeline_progress"] = pipeline_progress - if "transfer_label" in sig.parameters: - kwargs["transfer_label"] = transfer_label - if "progress_callback" in sig.parameters: - - def _cb(done, total): - try: - total_val = int(total) if total is not None else None - except Exception: - total_val = None - try: - if int(done or 0) <= 0: - pipeline_progress.begin_transfer( - label=transfer_label, - total=total_val, - ) - except Exception: - pass - try: - pipeline_progress.update_transfer( - label=transfer_label, - completed=int(done or 0), - total=total_val, - ) - except Exception: - pass - - kwargs["progress_callback"] = _cb - - downloaded = downloader(str(file_hash), **kwargs) - - if isinstance(downloaded, Path) and downloaded.exists(): - if output_dir is not None: - pipe_obj.is_temp = False - if isinstance(pipe_obj.extra, dict): - pipe_obj.extra["_direct_export_download"] = True - else: - pipe_obj.extra = {"_direct_export_download": True} - return downloaded, None - pipe_obj.is_temp = True - return downloaded, tmp_dir - except Exception: - pass - - if tmp_dir is not None: - try: - shutil.rmtree(tmp_dir, ignore_errors=True) - except Exception: - pass - return None, None - - @staticmethod - def _download_remote_backend_url( - remote_url: str, - pipe_obj: models.PipeObject, - *, - file_hash: Optional[str] = None, - output_dir: Optional[Path] = None, - ) -> Tuple[Optional[Path], Optional[Path]]: - """Best-effort fetch of a remote backend URL. - - Returns (downloaded_path, temp_dir_to_cleanup). - When ``output_dir`` is provided, the file is downloaded directly there and no - temp cleanup path is returned. - """ - - url_text = str(remote_url or "").strip() - if not url_text: - return None, None - if not url_text.lower().startswith(_REMOTE_URL_PREFIXES): - return None, None - # This helper performs generic HTTP downloads only. - # Non-HTTP schemes (e.g. hydrus://, tidal:) should be handled by - # plugin-specific resolvers via _maybe_download_plugin_result. - if not url_text.lower().startswith(("http://", "https://")): - return None, None - - tmp_dir: Optional[Path] = None - try: - download_root = output_dir - if download_root is None: - tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) - download_root = tmp_dir - - suggested_name = Add_File._build_provider_filename( - pipe_obj, - fallback_hash=file_hash, - source_url=url_text, - ) - pipeline_progress = PipelineProgress(ctx) - try: - destination_label = str(download_root) if download_root is not None else "temporary workspace" - pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}") - except Exception: - pass - - downloaded = download_direct_file( - url_text, - download_root, - quiet=False, - suggested_filename=suggested_name, - pipeline_progress=pipeline_progress, - ) - downloaded_path = getattr(downloaded, "path", None) - if isinstance(downloaded_path, Path) and downloaded_path.exists(): - if output_dir is not None: - pipe_obj.is_temp = False - if isinstance(pipe_obj.extra, dict): - pipe_obj.extra["_direct_export_download"] = True - else: - pipe_obj.extra = {"_direct_export_download": True} - return downloaded_path, None - - pipe_obj.is_temp = True - return downloaded_path, tmp_dir - except Exception: - pass - finally: - try: - PipelineProgress(ctx).clear_status() - except Exception: - pass - - if tmp_dir is not None: - try: - shutil.rmtree(tmp_dir, ignore_errors=True) - except Exception: - pass - return None, None - - @staticmethod - def _build_provider_filename( - pipe_obj: models.PipeObject, - fallback_hash: Optional[str] = None, - source_url: Optional[str] = None, - ) -> str: - title_candidates: List[str] = [] - title_value = getattr(pipe_obj, "title", "") - if title_value: - title_candidates.append(str(title_value)) - - extra = getattr(pipe_obj, "extra", {}) - if isinstance(extra, dict): - candid = extra.get("name") or extra.get("title") - if candid: - title_candidates.append(str(candid)) - - metadata = getattr(pipe_obj, "metadata", {}) - if isinstance(metadata, dict): - meta_name = metadata.get("title") or metadata.get("name") - if meta_name: - title_candidates.append(str(meta_name)) - - text = "" - for candidate in title_candidates: - if candidate: - text = candidate.strip() - if text: - break - - if not text and fallback_hash: - text = fallback_hash[:8] - - safe_name = sanitize_filename(text or "download") - - ext = "" - if isinstance(metadata, dict): - ext = metadata.get("ext") or metadata.get("extension") or "" - if not ext and isinstance(extra, dict): - ext = extra.get("ext") or "" - if not ext and source_url: - try: - parsed = urlparse(source_url) - ext = Path(parsed.path).suffix.lstrip(".") - except Exception: - ext = "" - - if ext: - ext_text = str(ext) - if not ext_text.startswith("."): - ext_text = "." + ext_text.lstrip(".") - if not safe_name.lower().endswith(ext_text.lower()): - safe_name = f"{safe_name}{ext_text}" - - return safe_name or "download" - - @staticmethod - def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]: - if not instance or not backend_name: - return None - try: - return instance[backend_name] - except Exception: - pass - target = str(backend_name or "").strip().lower() - if not target: - return None - try: - for candidate in instance.list_backends(): - if isinstance(candidate, str) and candidate.strip().lower() == target: - try: - return instance[candidate] - except Exception: - continue - except Exception: - pass - return None - - @staticmethod - def _resolve_source( - result: Any, - source_arg: Optional[str], - pipe_obj: models.PipeObject, - config: Dict[str, - Any], - export_destination: Optional[Path] = None, - store_instance: Optional[Any] = None, - deps: Optional[_CommandDependencies] = None, - ) -> Tuple[Optional[Path], - Optional[str], - Optional[Path]]: - """Resolve the source file path from the positional source arg or pipeline result. - - Returns (media_path, file_hash, temp_dir_to_cleanup). - """ - # PRIORITY 1a: Prefer an explicit local path when it exists. - # This avoids unnecessary backend.get_file(hash) probes (and remote fallbacks) - # for freshly downloaded temp files that are not yet present in the source store. - if isinstance(result, dict): - r_path = result.get("path") - r_hash = result.get("hash") - if r_path: - try: - p = coerce_to_path(r_path) - if p.exists() and p.is_file(): - pipe_obj.path = str(p) - return p, str(r_hash) if r_hash else None, None - except Exception: - pass - - # PRIORITY 1b: Try hash+store from result (fetch from backend) - r_hash = get_field(result, "hash") or get_field(result, "file_hash") - r_store = get_field(result, "store") - - if r_hash and r_store: - try: - if deps is None: - deps = _CommandDependencies(config) - backend_registry = store_instance or deps.get_backend_registry() - - backend = Add_File._resolve_backend_by_name(backend_registry, r_store) - if backend is not None: - mp = backend.get_file(r_hash) - if isinstance(mp, Path) and mp.exists(): - pipe_obj.path = str(mp) - return mp, str(r_hash), None - if isinstance(mp, str) and mp.strip(): - try: - mp_path = Path(str(mp)) - except Exception: - mp_path = None - if mp_path is not None and mp_path.exists() and mp_path.is_file(): - pipe_obj.path = str(mp_path) - return mp_path, str(r_hash), None - - dl_path, tmp_dir = Add_File._maybe_download_backend_file( - backend, - str(r_hash), - pipe_obj, - output_dir=export_destination, - ) - if dl_path and dl_path.exists(): - pipe_obj.path = str(dl_path) - return dl_path, str(r_hash), tmp_dir - - dl_path, tmp_dir = Add_File._download_remote_backend_url( - str(mp), - pipe_obj, - file_hash=str(r_hash), - output_dir=export_destination, - ) - if dl_path and dl_path.exists(): - pipe_obj.path = str(dl_path) - return dl_path, str(r_hash), tmp_dir - except Exception as exc: - debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}") - - # PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result) - candidate: Optional[Path] = None - - if source_arg: - candidate = Path(source_arg) - elif pipe_obj.path: - candidate = Path(pipe_obj.path) - - if not candidate: - # Unwrap list if needed - obj = result[0] if isinstance(result, list) and result else result - if obj: - try: - candidate = coerce_to_path(obj) - except ValueError: - pass - - if candidate: - s = str(candidate).lower() - if s.startswith(_REMOTE_URL_PREFIXES): - # For remote sources, prefer plugin-specific resolvers first - # (e.g. hydrus://), then generic HTTP fallback. - downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_plugin_result( - result, - pipe_obj, - config, - deps=deps, - ) - if downloaded_path: - pipe_obj.path = str(downloaded_path) - return downloaded_path, hash_hint, tmp_dir - - dl_path, tmp_dir = Add_File._download_remote_backend_url( - str(candidate), - pipe_obj, - file_hash=get_field(result, "hash") or get_field(result, "file_hash"), - output_dir=export_destination, - ) - if dl_path: - pipe_obj.path = str(dl_path) - hash_hint = get_field(result, "hash") or get_field(result, "file_hash") - return dl_path, hash_hint, tmp_dir - - log("add-file could not auto-fetch remote source. Use download-file first.", file=sys.stderr) - return None, None, None - - pipe_obj.path = str(candidate) - # Retain hash from input if available to avoid re-hashing - hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None) - return candidate, hash_hint, None - - downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_plugin_result( - result, - pipe_obj, - config, - deps=deps, - ) - if downloaded_path: - pipe_obj.path = str(downloaded_path) - return downloaded_path, hash_hint, tmp_dir - - debug(f"No resolution path matched. result type={type(result).__name__}") - log("File path could not be resolved") - return None, None, None - - @staticmethod - def _normalize_provider_key(value: Optional[Any]) -> Optional[str]: - if value is None: - return None - try: - normalized = str(value).strip().lower() - except Exception: - return None - if not normalized: - return None - if "." in normalized: - normalized = normalized.split(".", 1)[0] - return normalized - - @staticmethod - def validate_preflight_args( - args: Sequence[str], - config: Optional[Dict[str, Any]] = None, - ) -> Optional[str]: - from PluginCore.registry import PLUGIN_REGISTRY - - cfg = config if isinstance(config, dict) else {} - - if Add_File._uses_legacy_path_flag(args): - return f"Pipeline error: {Add_File._legacy_path_flag_message()}" - - try: - parsed = parse_cmdlet_args(args, CMDLET) - except Exception as exc: - return f"Pipeline error: invalid add-file arguments: {exc}" - - deps = _CommandDependencies(cfg) - storage_registry: Optional[BackendRegistry] = None - - location = parsed.get("instance") - plugin_instance = parsed.get("instance") - plugin_name = parsed.get("plugin") - - is_storage_backend_location = False - if location and not plugin_name: - try: - backend_registry_for_lookup = storage_registry or deps.get_backend_registry() - storage_registry = backend_registry_for_lookup - is_storage_backend_location = Add_File._resolve_backend_by_name( - backend_registry_for_lookup, - str(location), - ) is not None - except Exception: - is_storage_backend_location = False - - if location and not plugin_name and not is_storage_backend_location: - resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( - location, - cfg, - deps=deps, - require_explicit=True, - ) - if resolved_local_path: - return None - return ( - f"Pipeline error: storage backend '{location}' not found. " - "Use -plugin local -instance <name|path> for local export or configure that store backend." - ) - - normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) - if normalized_plugin_name: - upload_plugin = deps.get_plugin_for_cmdlet(normalized_plugin_name, "add-file") - if upload_plugin is None: - plugin_info = PLUGIN_REGISTRY.get(normalized_plugin_name) - if plugin_info is not None: - canonical_plugin_name = str(plugin_info.canonical_name or normalized_plugin_name).strip().lower() - if canonical_plugin_name == "loc": - return ( - "Pipeline error: plugin 'loc' does not support add-file. " - "Use -plugin local -instance <name|path> for local export." - ) - if "add-file" not in plugin_info.supported_cmdlets: - return f"Pipeline error: plugin '{canonical_plugin_name}' does not support add-file." - return f"Pipeline error: plugin '{canonical_plugin_name}' is not configured or not available for add-file." - return f"Pipeline error: unknown add-file plugin '{plugin_name}'." - - if normalized_plugin_name == "local": - requested_local = str(plugin_instance or location or "").strip() or "<default>" - resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( - plugin_instance or location, - cfg, - deps=deps, - require_explicit=bool(plugin_instance or location), - ) - if not resolved_local_path: - return ( - f"Pipeline error: local destination '{requested_local}' is not configured. " - "Use -plugin local -instance <name|path>." - ) - - return None - - @staticmethod - def _resolve_plugin_storage_backend( - plugin_name: Optional[Any], - instance_name: Optional[Any], - config: Dict[str, Any], - *, - store_instance: Optional[Any] = None, - deps: Optional[_CommandDependencies] = None, - ) -> Optional[str]: - plugin_key = Add_File._normalize_provider_key(plugin_name) - if not plugin_key: - return None - - if deps is None: - deps = _CommandDependencies(config) - - file_provider = deps.get_plugin_for_cmdlet(plugin_key, "add-file") - if file_provider is None: - return None - - resolver = getattr(file_provider, "resolve_backend", None) - if not callable(resolver): - return None - - explicit_instance = str(instance_name or "").strip() or None - try: - backend_registry = store_instance if store_instance is not None else deps.get_backend_registry() - except Exception: - backend_registry = None - - try: - resolved_name, backend = resolver( - explicit_instance, - storage=backend_registry, - require_explicit=bool(explicit_instance), - ) - except TypeError: - try: - resolved_name, backend = resolver(explicit_instance) - except Exception: - return None - except Exception: - return None - - if backend is None: - return None - - resolved_text = str(resolved_name or explicit_instance or "").strip() - if not resolved_text: - return None - - checker = getattr(file_provider, "is_backend", None) - if callable(checker): - try: - if not checker(backend, resolved_text): - return None - except Exception: - return None - - return resolved_text - - @staticmethod - def _resolve_local_export_plugin_target( - requested: Optional[Any], - config: Dict[str, Any], - *, - deps: Optional[_CommandDependencies] = None, - require_explicit: bool = False, - ) -> tuple[Optional[str], Optional[str]]: - if deps is None: - deps = _CommandDependencies(config) - - file_provider = deps.get_plugin_for_cmdlet("local", "add-file") - if file_provider is None: - return None, None - - resolver = getattr(file_provider, "resolve_destination", None) - if not callable(resolver): - return None, None - - requested_text = str(requested or "").strip() or None - try: - resolved_name, settings = resolver( - requested_text, - require_explicit=require_explicit, - ) - except TypeError: - try: - resolved_name, settings = resolver(requested_text) - except Exception: - return None, None - except Exception: - return None, None - - path_value = str((settings or {}).get("path") or "").strip() - if not path_value: - return None, None - resolved_text = str(resolved_name or requested_text or "").strip() or None - return resolved_text, path_value - - @staticmethod - def _maybe_download_plugin_result( - result: Any, - pipe_obj: models.PipeObject, - config: Dict[str, Any], - deps: Optional[_CommandDependencies] = None, - ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: - plugin_key = None - for source in ( - pipe_obj.plugin, - get_field(result, "plugin"), - get_field(result, "table"), - ): - candidate = Add_File._normalize_provider_key(source) - if candidate: - plugin_key = candidate - break - - if not plugin_key: - return None, None, None - - if deps is None: - deps = _CommandDependencies(config) - - plugin = deps.get_plugin(plugin_key) - if plugin is None: - return None, None, None - - try: - return plugin.resolve_pipe_result_download(result, pipe_obj) - except Exception as exc: - debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}") - - return None, None, None - - @staticmethod - def _download_piped_source( - pipe_obj: models.PipeObject, - config: Dict[str, Any], - store_instance: Optional[Any], - deps: Optional[_CommandDependencies] = None, - ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: - r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip() - r_store = str(getattr(pipe_obj, "store", None) or "").strip() - if not (r_hash and r_store): - return None, None, None - - if deps is None: - deps = _CommandDependencies(config) - - backend_registry = store_instance or deps.get_backend_registry() - backend = Add_File._resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None - if backend is None: - return None, None, None - - try: - source = backend.get_file(r_hash.lower()) - if isinstance(source, Path) and source.exists(): - pipe_obj.path = str(source) - return source, str(r_hash), None - if isinstance(source, str) and source.strip(): - dl_path, tmp_dir = Add_File._maybe_download_backend_file( - backend, str(r_hash), pipe_obj - ) - if dl_path and dl_path.exists(): - return dl_path, str(r_hash), tmp_dir - source_url = str(source).strip() - if source_url.lower().startswith(("http://", "https://")): - download_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) - try: - filename = Add_File._build_provider_filename( - pipe_obj, - str(r_hash), - source_url, - ) - downloaded = download_direct_file( - source_url, - download_dir, - quiet=True, - suggested_filename=filename, - ) - downloaded_path = downloaded.path - if downloaded_path and downloaded_path.exists(): - pipe_obj.is_temp = True - pipe_obj.path = str(downloaded_path) - return downloaded_path, str(r_hash), download_dir - except Exception as exc: - debug(f"[add-file] Provider download failed: {exc}") - try: - shutil.rmtree(download_dir, ignore_errors=True) - except Exception: - pass - except Exception: - pass - - return None, None, None - - @staticmethod - def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]: - """Scan a directory for supported media files and return list of file info dicts. - - Each dict contains: - - path: Path object - - name: filename - - hash: sha256 hash (or None if compute_hash=False) - - size: file size in bytes - - ext: file extension - """ - if not directory.exists() or not directory.is_dir(): - return [] - - files_info: List[Dict[str, Any]] = [] - - try: - for item in directory.iterdir(): - if not item.is_file(): - continue - - ext = item.suffix.lower() - if ext not in SUPPORTED_MEDIA_EXTENSIONS: - continue - - file_hash = None - # Compute hash if requested (computing can be expensive for large dirs) - if compute_hash: - try: - file_hash = sha256_file(item) - except Exception as exc: - debug(f"Failed to hash {item}: {exc}") - # If hashing is required, skip this file; otherwise include without hash - continue - - # Get file size - try: - size = item.stat().st_size - except Exception: - size = 0 - - files_info.append( - { - "path": item, - "name": item.name, - "hash": file_hash, - "size": size, - "ext": ext, - } - ) - except Exception as exc: - debug(f"Error scanning directory {directory}: {exc}") - - return files_info - - @staticmethod - def _validate_source(media_path: Optional[Path], allow_all_extensions: bool = False) -> bool: - """Validate that the source file exists and is supported. - - Args: - media_path: Path to the file to validate - allow_all_extensions: If True, skip file type filtering for non-backend exports. - If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest. - """ - if media_path is None: - return False - - if not media_path.exists() or not media_path.is_file(): - log(f"File not found: {media_path}") - return False - - # Validate file type only when ingesting into a storage backend. - if not allow_all_extensions: - file_extension = media_path.suffix.lower() - if file_extension not in SUPPORTED_MEDIA_EXTENSIONS: - log(f"❌ Unsupported file type: {file_extension}", file=sys.stderr) - return False - - return True - - @staticmethod - def _is_probable_url(s: Any) -> bool: - """Check if a string looks like a URL/magnet/identifier (vs a tag/title).""" - if not isinstance(s, str): - return False - val = s.strip().lower() - if not val: - return False - # Obvious schemes - if val.startswith(_REMOTE_URL_PREFIXES): - return True - # Domain-like patterns or local file paths (but we want URLs here) - if "://" in val: - return True - # Hydrus hash-like search queries are NOT urls - if val.startswith("hash:"): - return False - return False - - @staticmethod - def _get_url(result: Any, pipe_obj: models.PipeObject) -> List[str]: - """Extract valid URLs from pipe object or result dict.""" - from SYS.metadata import normalize_urls - - candidates: List[str] = [] - - # 1. Prefer explicit PipeObject top-level field - if pipe_obj.url: - candidates.append(pipe_obj.url) - if pipe_obj.source_url: - candidates.append(pipe_obj.source_url) - - # 2. Check extra and metadata fields - if isinstance(pipe_obj.extra, dict): - u = pipe_obj.extra.get("url") - if isinstance(u, list): - candidates.extend(str(x) for x in u if x) - elif isinstance(u, str): - candidates.append(u) - - # 3. Check result (which might be a dict or another PipeObject) - raw_from_result = extract_url_from_result(result) - if raw_from_result: - candidates.extend(raw_from_result) - - # 4. Normalize and filter: MUST look like a URL to avoid tag leakage - normalized = normalize_urls(candidates) - return [u for u in normalized if Add_File._is_probable_url(u)] - - @staticmethod - def _get_relationships(result: Any, - pipe_obj: models.PipeObject) -> Optional[Dict[str, - Any]]: - try: - rels = pipe_obj.get_relationships() - if rels: - return rels - except Exception: - pass - if isinstance(result, dict) and result.get("relationships"): - return result.get("relationships") - try: - return extract_relationships(result) - except Exception: - return None - - @staticmethod - def _get_duration(result: Any, pipe_obj: models.PipeObject) -> Optional[float]: - - def _parse_duration(value: Any) -> Optional[float]: - if value is None: - return None - if isinstance(value, (int, float)): - return float(value) if value > 0 else None - if isinstance(value, str): - s = value.strip() - if not s: - return None - try: - candidate = float(s) - return candidate if candidate > 0 else None - except ValueError: - pass - if ":" in s: - parts = [p.strip() for p in s.split(":") if p.strip()] - if len(parts) in {2, - 3} and all(p.isdigit() for p in parts): - nums = [int(p) for p in parts] - if len(nums) == 2: - minutes, seconds = nums - return float(minutes * 60 + seconds) - hours, minutes, seconds = nums - return float(hours * 3600 + minutes * 60 + seconds) - return None - - parsed = _parse_duration(getattr(pipe_obj, "duration", None)) - if parsed is not None: - return parsed - try: - return _parse_duration(extract_duration(result)) - except Exception: - return None - - @staticmethod - def _get_note_text(result: Any, - pipe_obj: models.PipeObject, - note_name: str) -> Optional[str]: - """Extract a named note text from a piped item. - - Supports: - - pipe_obj.extra["notes"][note_name] - - result["notes"][note_name] for dict results - - pipe_obj.extra[note_name] / result[note_name] as fallback - """ - - def _normalize(val: Any) -> Optional[str]: - if val is None: - return None - if isinstance(val, bytes): - try: - val = val.decode("utf-8", errors="ignore") - except Exception: - val = str(val) - if isinstance(val, str): - text = val.strip() - return text if text else None - try: - text = str(val).strip() - return text if text else None - except Exception: - return None - - note_key = str(note_name or "").strip() - if not note_key: - return None - - # Prefer notes dict on PipeObject.extra (common for cmdlet-emitted dicts) - try: - if isinstance(pipe_obj.extra, dict): - notes_val = pipe_obj.extra.get("notes") - if isinstance(notes_val, dict) and note_key in notes_val: - return _normalize(notes_val.get(note_key)) - if note_key in pipe_obj.extra: - return _normalize(pipe_obj.extra.get(note_key)) - except Exception: - pass - - # Fallback to raw result dict - if isinstance(result, dict): - try: - notes_val = result.get("notes") - if isinstance(notes_val, dict) and note_key in notes_val: - return _normalize(notes_val.get(note_key)) - if note_key in result: - return _normalize(result.get(note_key)) - except Exception: - pass - - return None - - @staticmethod - def _update_pipe_object_destination( - pipe_obj: models.PipeObject, - *, - hash_value: str, - store: str, - plugin: Optional[str] = None, - path: Optional[str], - tag: List[str], - title: Optional[str], - extra_updates: Optional[Dict[str, - Any]] = None, - ) -> None: - pipe_obj.hash = hash_value - pipe_obj.store = store - pipe_obj.plugin = plugin - pipe_obj.is_temp = False - pipe_obj.path = path - pipe_obj.tag = tag - if title: - pipe_obj.title = title - if isinstance(pipe_obj.extra, dict): - pipe_obj.extra.update(extra_updates or {}) - else: - pipe_obj.extra = dict(extra_updates or {}) - - @staticmethod - def _emit_pipe_object(pipe_obj: models.PipeObject) -> None: - payload = pipe_obj.to_dict() - ctx.emit(payload) - ctx.set_current_stage_table(None) - - stage_ctx = ctx.get_stage_context() - is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) - if not is_last: - return - - try: - Add_File._stop_live_progress_for_terminal_render() - from .._shared import display_and_persist_items - - display_and_persist_items([payload], title="Result", subject=payload) - except Exception: - pass - - @staticmethod - def _stop_live_progress_for_terminal_render() -> None: - try: - live_progress = ctx.get_live_progress() - except Exception: - live_progress = None - - if live_progress is None: - return - - try: - stage_ctx = ctx.get_stage_context() - pipe_idx = getattr(stage_ctx, "pipe_index", None) - if isinstance(pipe_idx, int): - live_progress.finish_pipe(int(pipe_idx), force_complete=True) - except Exception: - pass - - try: - live_progress.stop() - except Exception: - pass - - try: - if hasattr(ctx, "set_live_progress"): - ctx.set_live_progress(None) - except Exception: - pass - - @staticmethod - def _emit_storage_result( - payload: Dict[str, - Any], - *, - overlay: bool = True, - emit: bool = True - ) -> None: - """Emit a storage-style result payload. - - - Always emits the dict downstream (when in a pipeline). - - If this is the last stage (or not in a pipeline), prints a search-file-like table - and sets an overlay table/items for @N selection. - """ - # Emit for downstream commands (no-op if not in a pipeline) - if emit: - ctx.emit(payload) - - stage_ctx = ctx.get_stage_context() - is_last = (stage_ctx - is None) or bool(getattr(stage_ctx, - "is_last_stage", - False)) - if not is_last or not overlay: - return - - try: - from SYS.result_table import Table - - table = Table("Result") - table.add_result(payload) - # Overlay so @1 refers to this add-file result without overwriting search history - publish_result_table(ctx, table, [payload], subject=payload, overlay=True) - except Exception: - # If table rendering fails, still keep @ selection items - try: - ctx.set_last_result_items_only([payload]) - except Exception: - pass - - @staticmethod - def _try_emit_search_file_by_hash( - *, - instance: str, - hash_value: str, - config: Dict[str, - Any] - ) -> Optional[List[Any]]: - """Run search-file for a single hash so the final table/payload is consistent. - - Important: `add-file` is treated as an action command by the CLI, so the CLI only - prints tables for it when a display overlay exists. After running search-file, - this copies the resulting table into the display overlay (when this is the last - stage) so the canonical store table is what the user sees and can select from. - - Returns the emitted search-file payload items on success, else None. - """ - try: - from cmdlet.file.search import CMDLET as search_file_cmdlet - - args = ["-instance", str(instance), f"hash:{str(hash_value)}"] - - # Run search-file under a temporary stage context so its ctx.emit() calls - # don't interfere with the outer add-file pipeline stage. - prev_ctx = ctx.get_stage_context() - temp_ctx = ctx.PipelineStageContext( - stage_index=0, - total_stages=1, - pipe_index=0, - worker_id=getattr(prev_ctx, - "worker_id", - None), - ) - ctx.set_stage_context(temp_ctx) - try: - code = search_file_cmdlet.run(None, args, config) - emitted_items = list(getattr(temp_ctx, "emits", []) or []) - finally: - ctx.set_stage_context(prev_ctx) - if code != 0: - return None - - # Promote the search-file result to a display overlay so the CLI prints it - # for action commands like add-file. - stage_ctx = ctx.get_stage_context() - is_last = (stage_ctx - is None) or bool(getattr(stage_ctx, - "is_last_stage", - False)) - if is_last: - try: - table = ctx.get_last_result_table() - items = ctx.get_last_result_items() - overlay_existing_result_table( - ctx, - subject={ - "store": instance, - "hash": hash_value - }, - ) - except Exception: - pass - - return emitted_items - except Exception as exc: - debug( - f"[add-file] Failed to run search-file after add-file: {type(exc).__name__}: {exc}" - ) - return None - - @staticmethod - def _prepare_metadata( - result: Any, - media_path: Path, - pipe_obj: models.PipeObject, - config: Dict[str, - Any], - ) -> Tuple[List[str], - List[str], - Optional[str], - Optional[str]]: - """ - Prepare tags, url, and title for the file. - Returns (tags, url, preferred_title, file_hash) - """ - tags_from_result = list(pipe_obj.tag or []) - if not tags_from_result: - try: - tags_from_result = list(extract_tag_from_result(result) or []) - except Exception: - tags_from_result = [] - - url_from_result = Add_File._get_url(result, pipe_obj) - - def _has_namespace_tag(tags: Sequence[str], namespace: str) -> bool: - namespace_text = str(namespace or "").strip().lower() - if not namespace_text: - return False - prefix = f"{namespace_text}:" - for tag in tags or []: - text = str(tag or "").strip().lower() - if text.startswith(prefix): - return True - return False - - def _extract_screenshot_time_title() -> tuple[Optional[str], Optional[str]]: - # MPV screenshot saves use <title>_<2m46s>.png as a temp filename. - # When we ingest that into a tag-capable backend, recover the clean title - # and surface the capture position as a namespace tag instead of baking it - # into the stored display title. - current_title = str(preferred_title or "").strip() - filename_title = str(media_path.stem or "").strip() - if current_title and current_title != filename_title: - return None, None - if not url_from_result: - return None, None - suffix = str(media_path.suffix or "").strip().lower() - if suffix not in {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif", ".mhtml"}: - return None, None - match = _SCREENSHOT_TIME_SUFFIX_RE.match(str(media_path.stem or "").strip()) - if not match: - return None, None - title_text = str(match.group("title") or "").strip().replace("_", " ").strip() - label_text = str(match.group("label") or "").strip().lower() - if not title_text or not label_text: - return None, None - return title_text, label_text - - preferred_title = pipe_obj.title - if not preferred_title: - for t in tags_from_result: - if str(t).strip().lower().startswith("title:"): - candidate = t.split(":", 1)[1].strip().replace("_", " ").strip() - if candidate: - preferred_title = candidate - break - if not preferred_title: - preferred_title = extract_title_from_result(result) - if preferred_title: - preferred_title = preferred_title.replace("_", " ").strip() - - derived_screenshot_title, derived_time_tag = _extract_screenshot_time_title() - if derived_screenshot_title and ( - not preferred_title or str(preferred_title or "").strip() == str(media_path.stem or "").strip() - ): - preferred_title = derived_screenshot_title - - store = getattr(pipe_obj, "store", None) - _, sidecar_hash, sidecar_tags, sidecar_url = Add_File._load_sidecar_bundle( - media_path, store, config - ) - - def normalize_title_tag(tag: str) -> str: - if str(tag).strip().lower().startswith("title:"): - parts = tag.split(":", 1) - if len(parts) == 2: - value = parts[1].replace("_", " ").strip() - return f"title:{value}" - return tag - - tags_from_result_no_title = [ - t for t in tags_from_result - if not str(t).strip().lower().startswith("title:") - ] - sidecar_tags = collapse_namespace_tags( - [normalize_title_tag(t) for t in sidecar_tags], - "title", - prefer="last" - ) - sidecar_tags_filtered = [ - t for t in sidecar_tags if not str(t).strip().lower().startswith("title:") - ] - - merged_tags = merge_sequences( - tags_from_result_no_title, - sidecar_tags_filtered, - case_sensitive=True - ) - - if derived_time_tag and not _has_namespace_tag(merged_tags, "time") and not _has_namespace_tag(merged_tags, "timestamp"): - merged_tags.append(f"time:{derived_time_tag}") - - if preferred_title: - merged_tags.append(f"title:{preferred_title}") - - merged_url = merge_sequences(url_from_result, sidecar_url, case_sensitive=False) - # Final safety filter: ensures no tags/titles leaked into URL list - merged_url = [u for u in merged_url if Add_File._is_probable_url(u)] - - file_hash = Add_File._resolve_file_hash( - result, - media_path, - pipe_obj, - sidecar_hash - ) - - # Relationships must not be stored as tags. - # If relationship tags exist (legacy sidecar format), capture them into PipeObject.relationships - # and strip them from the final tag list. - relationship_tags = [ - t for t in merged_tags - if isinstance(t, str) and t.strip().lower().startswith("relationship:") - ] - if relationship_tags: - try: - if (not isinstance(getattr(pipe_obj, - "relationships", - None), - dict) or not pipe_obj.relationships): - king: Optional[str] = None - alts: List[str] = [] - for rel_tag in relationship_tags: - k, a = Add_File._parse_relationship_tag_king_alts(rel_tag) - if k and not king: - king = k - if a: - alts.extend(a) - if king: - seen_alt: set[str] = set() - alts = [ - h for h in alts if h and h != king and len(h) == 64 - and not (h in seen_alt or seen_alt.add(h)) - ] - payload: Dict[str, - Any] = { - "king": [king] - } - if alts: - payload["alt"] = alts - pipe_obj.relationships = payload - except Exception: - pass - - merged_tags = [ - t for t in merged_tags if - not (isinstance(t, str) and t.strip().lower().startswith("relationship:")) - ] - - # Persist back to PipeObject - pipe_obj.tag = merged_tags - if preferred_title and not pipe_obj.title: - pipe_obj.title = preferred_title - if file_hash and not pipe_obj.hash: - pipe_obj.hash = file_hash - if isinstance(pipe_obj.extra, dict): - # Update (don't setdefault) to ensure URLs matched from sidecars or source stores are tracked - pipe_obj.extra["url"] = merged_url - return merged_tags, merged_url, preferred_title, file_hash - - @staticmethod - def _normalize_hash_candidate(value: Any) -> str: - text = str(value or "").strip().lower() - if len(text) != 64: - return "" - if any(ch not in "0123456789abcdef" for ch in text): - return "" - return text - - @staticmethod - def _find_existing_hash_by_urls( - backend: Any, - urls: Sequence[str], - ) -> Optional[str]: - """Best-effort duplicate detection by URL before ingesting file bytes.""" - url_candidates: List[str] = [] - for raw in urls or []: - text = str(raw or "").strip() - if not text or not Add_File._is_probable_url(text): - continue - if text not in url_candidates: - url_candidates.append(text) - - if not url_candidates: - return None - - lookup_exact = getattr(backend, "find_hashes_by_url", None) - if callable(lookup_exact): - for candidate_url in url_candidates: - try: - hashes = lookup_exact(candidate_url) or [] - except Exception: - continue - if not isinstance(hashes, (list, tuple, set)): - continue - for item in hashes: - normalized = Add_File._normalize_hash_candidate(item) - if normalized: - return normalized - - searcher = getattr(backend, "search", None) - if callable(searcher): - for candidate_url in url_candidates: - try: - hits = searcher(f"url:{candidate_url}", limit=1, minimal=True) or [] - except Exception: - continue - if not isinstance(hits, list) or not hits: - continue - hit = hits[0] - for key in ("hash", "file_hash", "sha256"): - normalized = Add_File._normalize_hash_candidate(get_field(hit, key)) - if normalized: - return normalized - - return None - - @staticmethod - def _emit_plugin_upload_payload( - upload_payload: Dict[str, Any], - plugin_name: str, - instance_name: Optional[str], - pipe_obj: models.PipeObject, - media_path: Path, - delete_after: bool, - ) -> int: - payload = dict(upload_payload or {}) - extra_updates: Dict[str, Any] = {} - raw_extra = payload.get("extra") - if isinstance(raw_extra, dict): - extra_updates.update(raw_extra) - - if plugin_name: - extra_updates.setdefault("plugin", plugin_name) - if instance_name: - extra_updates.setdefault("instance", instance_name) - - raw_urls = payload.get("url") - if isinstance(raw_urls, str): - url_values = [raw_urls.strip()] if raw_urls.strip() else [] - extra_updates["url"] = url_values - elif isinstance(raw_urls, (list, tuple, set)): - url_values = [str(item).strip() for item in raw_urls if str(item).strip()] - extra_updates["url"] = url_values - - relationships = payload.get("relationships") - if relationships: - try: - pipe_obj.relationships = relationships - except Exception: - pass - - tags = payload.get("tag") - if isinstance(tags, list): - tag_values = [str(tag) for tag in tags] - else: - tag_values = list(pipe_obj.tag or []) - - title_value = str(payload.get("title") or pipe_obj.title or media_path.name).strip() or media_path.name - path_value = str(payload.get("path") or pipe_obj.path or media_path).strip() - hash_value = str( - payload.get("hash") - or payload.get("file_hash") - or getattr(pipe_obj, "hash", None) - or "unknown" - ).strip() or "unknown" - store_value = str(payload.get("store") or "").strip() - plugin_value = payload.get("plugin") - if plugin_value is None and plugin_name: - plugin_value = plugin_name - - Add_File._update_pipe_object_destination( - pipe_obj, - hash_value=hash_value, - store=store_value, - plugin=str(plugin_value) if plugin_value else None, - path=path_value, - tag=tag_values, - title=title_value, - extra_updates=extra_updates, - ) - Add_File._emit_pipe_object(pipe_obj) - - Add_File._cleanup_after_success(media_path, delete_source=delete_after) - return 0 - - @staticmethod - def _handle_plugin_upload( - media_path: Path, - plugin_name: str, - instance_name: Optional[str], - pipe_obj: models.PipeObject, - config: Dict[str, - Any], - delete_after: bool, - folder_name: Optional[str] = None, - ) -> int: - """Handle uploading via an add-file plugin (e.g. 0x0).""" - from PluginCore.registry import ( - PLUGIN_REGISTRY, - get_plugin_for_cmdlet, - list_plugins_for_cmdlet, - ) - - try: - file_provider = get_plugin_for_cmdlet(plugin_name, "add-file", config) - if not file_provider: - available_map = list_plugins_for_cmdlet("add-file", config) - available_add_plugins = [name for name, enabled in available_map.items() if enabled] - requested_plugin_info = PLUGIN_REGISTRY.get(plugin_name) - - if requested_plugin_info is not None and "add-file" in requested_plugin_info.supported_cmdlets: - show_plugin_config_panel([requested_plugin_info.canonical_name]) - else: - log(f"Add-file plugin '{plugin_name}' is not available or does not support add-file", file=sys.stderr) - - if available_add_plugins: - show_available_plugins_panel(sorted(available_add_plugins)) - return 1 - - upload_kwargs: Dict[str, Any] = { - "pipe_obj": pipe_obj, - "instance": instance_name, - } - pipeline_progress = PipelineProgress(ctx) - normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) - f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None) - if normalized_plugin_name == "local": - result = None - tags, urls, title, f_hash = Add_File._prepare_metadata(result, media_path, pipe_obj, config) - relationships = Add_File._get_relationships(result, pipe_obj) - direct_export_download = False - try: - if isinstance(pipe_obj.extra, dict): - direct_export_download = bool(pipe_obj.extra.pop("_direct_export_download", False)) - except Exception: - direct_export_download = False - - upload_kwargs.update( - { - "title": title, - "tags": tags, - "urls": urls, - "hash_value": f_hash, - "relationships": relationships, - "direct_export_download": direct_export_download, - "pipeline_progress": pipeline_progress, - } - ) - if folder_name is not None and str(folder_name).strip(): - upload_kwargs["folder_name"] = str(folder_name).strip() - - upload_result = file_provider.upload( - str(media_path), - **upload_kwargs, - ) - - duplicate_upload = False - duplicate_rule = "" - duplicate_target = "" - try: - if isinstance(getattr(pipe_obj, "extra", None), dict): - duplicate_upload = bool(pipe_obj.extra.get("upload_duplicate")) - duplicate_rule = str(pipe_obj.extra.get("upload_duplicate_rule") or "").strip() - duplicate_target = str(pipe_obj.extra.get("upload_duplicate_target") or "").strip() - except Exception: - duplicate_upload = False - duplicate_rule = "" - duplicate_target = "" - - except Exception as exc: - log(f"Upload failed: {exc}", file=sys.stderr) - return 1 - - if isinstance(upload_result, dict): - return Add_File._emit_plugin_upload_payload( - upload_result, - plugin_name, - instance_name, - pipe_obj, - media_path, - delete_after, - ) - - hoster_url = str(upload_result or "").strip() - - # Update PipeObject and emit - extra_updates: Dict[str, - Any] = { - "plugin": plugin_name, - "instance": instance_name, - "plugin_url": hoster_url, - } - if isinstance(pipe_obj.extra, dict): - # Also track hoster URL as a url for downstream steps - existing_known = list(pipe_obj.extra.get("url") or []) - if hoster_url and hoster_url not in existing_known: - existing_known.append(hoster_url) - extra_updates["url"] = existing_known - - file_path = pipe_obj.path or (str(media_path) if media_path else None) or "" - Add_File._update_pipe_object_destination( - pipe_obj, - hash_value=f_hash or "unknown", - store="", - plugin=plugin_name or None, - path=file_path, - tag=pipe_obj.tag, - title=pipe_obj.title or (media_path.name if media_path else None), - extra_updates=extra_updates, - ) - Add_File._emit_pipe_object(pipe_obj) - - Add_File._cleanup_after_success(media_path, delete_source=delete_after) - return 0 - - @staticmethod - def _handle_storage_backend( - result: Any, - media_path: Path, - backend_name: str, - pipe_obj: models.PipeObject, - config: Dict[str, - Any], - delete_after: bool, - *, - collect_payloads: Optional[List[Dict[str, - Any]]] = None, - collect_relationship_pairs: Optional[Dict[str, - set[tuple[str, - str]]]] = None, - defer_url_association: bool = False, - pending_url_associations: Optional[Dict[str, - List[tuple[str, - List[str]]]]] = None, - defer_tag_association: bool = False, - pending_tag_associations: Optional[Dict[str, - List[tuple[str, - List[str]]]]] = None, - suppress_last_stage_overlay: bool = False, - auto_search_file: bool = True, - store_instance: Optional[BackendRegistry] = None, - ) -> int: - """Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.).""" - ##log(f"Adding file to storage backend '{backend_name}': {media_path.name}", file=sys.stderr) - pipeline_progress = PipelineProgress(ctx) - - def _set_status(text: str) -> None: - try: - pipeline_progress.set_status(f"{backend_name}: {text}") - except Exception: - pass - - def _clear_status() -> None: - try: - pipeline_progress.clear_status() - except Exception: - pass - - delete_after_effective = bool(delete_after) - # ... (lines omitted for brevity but I need to keep them contextually correct) - if not delete_after_effective: - # When download-media is piped into add-file, the downloaded artifact is a temp file. - # After it is persisted to a storage backend, delete the temp copy to avoid duplicates. - try: - if (str(backend_name or "").strip().lower() != "temp" - and getattr(pipe_obj, - "is_temp", - False) - and getattr(pipe_obj, - "action", - None) == "cmdlet:download-media"): - from SYS.config import resolve_output_dir - - temp_dir = resolve_output_dir(config) - try: - if media_path.resolve().is_relative_to( - temp_dir.expanduser().resolve()): - delete_after_effective = True - debug( - f"[add-file] Auto-delete temp source after ingest: {media_path}" - ) - except Exception: - # If path resolution fails, fall back to non-destructive behavior - pass - except Exception: - pass - - try: - backend_registry = store_instance if store_instance is not None else BackendRegistry(config) - backend, backend_registry, backend_exc = sh.get_preferred_store_backend( - config, - backend_name, - store_registry=backend_registry, - suppress_debug=True, - ) - if backend is None: - raise backend_exc or KeyError(f"Unknown store backend: {backend_name}") - - # Use backend properties to drive metadata deferral behavior. - is_remote_backend = getattr(backend, "is_remote", False) - prefer_defer_tags = getattr(backend, "prefer_defer_tags", False) - supports_url_association = bool(getattr(backend, "supports_url_association", False)) - supports_note_association = bool(getattr(backend, "supports_note_association", False)) - supports_relationship_association = bool(getattr(backend, "supports_relationship_association", False)) - - # ... - # Prepare metadata from pipe_obj and sidecars - tags, url, title, f_hash = Add_File._prepare_metadata( - result, media_path, pipe_obj, config - ) - - # If we're moving/copying from one store to another, also copy the source store's - # existing associated URLs so they aren't lost. - try: - from SYS.metadata import normalize_urls - - source_store = None - source_hash = None - if isinstance(result, dict): - source_store = result.get("store") - source_hash = result.get("hash") - if not source_store: - source_store = getattr(pipe_obj, "store", None) - if not source_hash: - source_hash = getattr(pipe_obj, "hash", None) - if (not source_hash) and isinstance(pipe_obj.extra, dict): - source_hash = pipe_obj.extra.get("hash") - - source_store = str(source_store or "").strip() - source_hash = str(source_hash or "").strip().lower() - if (source_store and source_hash and len(source_hash) == 64 - and source_store.lower() != str(backend_name or "" - ).strip().lower()): - source_backend = None - try: - if source_store in store.list_backends(): - source_backend = store[source_store] - except Exception: - source_backend = None - - if source_backend is not None: - try: - src_urls = normalize_urls( - source_backend.get_url(source_hash) or [] - ) - except Exception: - src_urls = [] - - try: - dst_urls = normalize_urls(url or []) - except Exception: - dst_urls = [] - - merged: list[str] = [] - seen: set[str] = set() - for u in list(dst_urls or []) + list(src_urls or []): - if not u: - continue - if u in seen: - continue - seen.add(u) - merged.append(u) - url = merged - except Exception: - pass - - # Collect relationship pairs for post-ingest DB/API persistence. - if collect_relationship_pairs is not None and supports_relationship_association: - rels = Add_File._get_relationships(result, pipe_obj) - if isinstance(rels, dict) and rels: - king_hash, alt_hashes = Add_File._parse_relationships_king_alts(rels) - if king_hash and alt_hashes: - bucket = collect_relationship_pairs.setdefault( - str(backend_name), - set() - ) - for alt_hash in alt_hashes: - if alt_hash and alt_hash != king_hash: - bucket.add((alt_hash, king_hash)) - - # Relationships must never be stored as tags. - if isinstance(tags, list) and tags: - tags = [ - t for t in tags if not ( - isinstance(t, str) - and t.strip().lower().startswith("relationship:") - ) - ] - - # Auto-tag (best-effort) BEFORE uploading so tags land with the stored file. - try: - tags = _maybe_apply_florencevision_tags(media_path, list(tags or []), config, pipe_obj=pipe_obj) - pipe_obj.tag = list(tags or []) - except Exception as exc: - # strict mode raises from helper; treat here as a hard failure - log(f"[add-file] FlorenceVision tagging error: {exc}", file=sys.stderr) - return 1 - - upload_tags = tags - if prefer_defer_tags and upload_tags: - upload_tags = [] - try: - debug_panel( - "add-file store", - [ - ("backend", backend_name), - ("path", media_path), - ("title", title), - ("hash_hint", f_hash[:12] if f_hash else "N/A"), - ("defer_tags", bool(prefer_defer_tags and tags)), - ], - border_style="yellow", - ) - except Exception: - pass - - duplicate_hash = Add_File._find_existing_hash_by_urls(backend, url) - if duplicate_hash: - debug( - f"[add-file] URL duplicate detected in '{backend_name}', skipping upload and reusing hash {duplicate_hash[:12]}..." - ) - file_identifier = duplicate_hash - else: - # Call backend's add_file with full metadata. - # Backend returns hash as identifier. If we already know the hash from _resolve_source - # (which came from download-file emit), pass it to skip re-hashing large files. - file_identifier = backend.add_file( - media_path, - title=title, - tag=upload_tags, - url=[] if ((defer_url_association and url) or (not supports_url_association)) else url, - file_hash=f_hash, - pipeline_progress=pipeline_progress, - transfer_label=title or media_path.name, - ) - ##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr) - - stored_path: Optional[str] = None - # IMPORTANT: avoid calling get_file() for remote backends by default to avoid - # unintended network activity or credential exposure in result payloads. - try: - if not is_remote_backend: - # For local backends, resolving the path is cheap and useful. - maybe_path = backend.get_file(file_identifier) - if isinstance(maybe_path, Path): - stored_path = str(maybe_path) - elif isinstance(maybe_path, str) and maybe_path: - stored_path = maybe_path - except Exception: - stored_path = None - - # Compute canonical hash value for downstream use (defensive against non-string returns). - if isinstance(file_identifier, str) and len(file_identifier) == 64: - chosen_hash = file_identifier - else: - chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown") - - Add_File._update_pipe_object_destination( - pipe_obj, - hash_value=chosen_hash, - store=backend_name, - path=stored_path, - tag=tags, - title=title or pipe_obj.title or media_path.name, - extra_updates={ - "url": url, - }, - ) - - # Emit a search-file-like payload for consistent tables and natural piping. - # Keep hash/store for downstream commands (get-tag, download-file, etc.). - resolved_hash = chosen_hash - - if prefer_defer_tags and tags: - # Support deferring tag application for batching bulk operations - if defer_tag_association and pending_tag_associations is not None: - try: - pending_tag_associations.setdefault(str(backend_name), []).append((str(resolved_hash), list(tags))) - except Exception: - pass - else: - try: - adder = getattr(backend, "add_tag", None) - if callable(adder): - _set_status("applying deferred tags") - adder(resolved_hash, list(tags)) - except Exception as exc: - log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr) - - # If we have url(s), ensure they get associated with the destination file. - # This mirrors `add-url` behavior but avoids emitting extra pipeline noise. - if url and supports_url_association: - if defer_url_association and pending_url_associations is not None: - try: - pending_url_associations.setdefault( - str(backend_name), - [] - ).append((str(resolved_hash), - list(url))) - except Exception: - pass - else: - try: - # Folder.add_file already persists URLs, avoid extra DB traffic here. - if not is_folder_backend: - _set_status("associating urls") - backend.add_url(resolved_hash, list(url)) - except Exception: - pass - - # If a subtitle note was provided upstream (e.g., download-media writes notes.sub), - # persist it automatically like add-note would. - def _write_note(note_name: str, note_text: Optional[str]) -> None: - if not note_text or not supports_note_association: - return - try: - setter = getattr(backend, "set_note", None) - if callable(setter): - _set_status(f"writing {note_name} note") - setter(resolved_hash, note_name, note_text) - except Exception as exc: - debug_panel( - "add-file note write failed", - [ - ("store", backend_name), - ("hash", resolved_hash), - ("note", note_name), - ("error", exc), - ], - border_style="yellow", - ) - - _write_note("sub", Add_File._get_note_text(result, pipe_obj, "sub")) - _write_note("lyric", Add_File._get_note_text(result, pipe_obj, "lyric")) - _write_note("chapters", Add_File._get_note_text(result, pipe_obj, "chapters")) - _write_note("caption", Add_File._get_note_text(result, pipe_obj, "caption")) - - meta: Dict[str, - Any] = {} - try: - if not is_folder_backend: - _set_status("loading stored metadata") - meta = backend.get_metadata(resolved_hash) or {} - except Exception: - meta = {} - - # Determine size bytes - size_bytes: Optional[int] = None - for key in ("size_bytes", "size", "filesize", "file_size"): - try: - raw_size = meta.get(key) - if raw_size is not None: - size_bytes = int(raw_size) - break - except Exception: - pass - if size_bytes is None: - try: - size_bytes = int(media_path.stat().st_size) - except Exception: - size_bytes = None - - # Determine title/ext - title_out = ( - meta.get("title") or title or pipe_obj.title or media_path.stem - or media_path.name - ) - ext_out = meta.get("ext") or media_path.suffix.lstrip(".") - - payload: Dict[ - str, - Any - ] = { - "title": title_out, - "ext": str(ext_out or ""), - "size_bytes": size_bytes, - "store": backend_name, - "hash": resolved_hash, - # Preserve extra fields for downstream commands (kept hidden by default table rules) - "path": stored_path, - "tag": list(tags or []), - "url": list(url or []), - } - if collect_payloads is not None: - try: - collect_payloads.append(payload) - except Exception: - pass - - # Keep the add-file 1-row summary overlay (when last stage), then emit the - # canonical search-file payload/table for piping/selection consistency. - if auto_search_file and resolved_hash and resolved_hash != "unknown": - # Show the add-file summary (overlay only) but let search-file provide the downstream payload. - Add_File._emit_storage_result( - payload, - overlay=not suppress_last_stage_overlay, - emit=False - ) - - refreshed_items = Add_File._try_emit_search_file_by_hash( - instance=backend_name, - hash_value=resolved_hash, - config=config, - ) - if refreshed_items: - # Re-emit the canonical store rows so downstream stages receive them. - for emitted in refreshed_items: - ctx.emit(emitted) - else: - # Fall back to emitting the add-file payload so downstream stages still receive an item. - ctx.emit(payload) - else: - Add_File._emit_storage_result( - payload, - overlay=not suppress_last_stage_overlay, - emit=True - ) - - Add_File._cleanup_after_success( - media_path, - delete_source=delete_after_effective - ) - _clear_status() - return 0 - - except Exception as exc: - _clear_status() - log( - f"❌ Failed to add file to backend '{backend_name}': {exc}", - file=sys.stderr - ) - import traceback - - traceback.print_exc(file=sys.stderr) - return 1 - - # --- Helpers --- - - @staticmethod - def _apply_pending_url_associations( - pending: Dict[str, - List[tuple[str, - List[str]]]], - config: Dict[str, - Any], - store_instance: Optional[BackendRegistry] = None, - ) -> None: - """Apply deferred URL associations in bulk, grouped per backend.""" - - try: - backend_registry = store_instance if store_instance is not None else BackendRegistry(config) - except Exception: - return - - for backend_name, pairs in (pending or {}).items(): - if not pairs: - continue - try: - backend, backend_registry, _exc = sh.get_store_backend( - config, - backend_name, - store_registry=backend_registry, - ) - if backend is None: - continue - - if not bool(getattr(backend, "supports_url_association", False)): - continue - - items = sh.coalesce_hash_value_pairs(pairs) - if not items: - continue - - bulk = getattr(backend, "add_url_bulk", None) - if callable(bulk): - try: - bulk(items) - continue - except Exception: - pass - - single = getattr(backend, "add_url", None) - if callable(single): - for h, u in items: - try: - single(h, u) - except Exception: - continue - except Exception: - continue - - @staticmethod - def _apply_pending_tag_associations( - pending: Dict[str, - List[tuple[str, - List[str]]]], - config: Dict[str, - Any], - store_instance: Optional[BackendRegistry] = None, - ) -> None: - """Apply deferred tag associations in bulk, grouped per backend.""" - - try: - backend_registry = store_instance if store_instance is not None else BackendRegistry(config) - except Exception: - return - - sh.run_store_hash_value_batches( - config, - pending or {}, - bulk_method_name="add_tags_bulk", - single_method_name="add_tag", - store_registry=backend_registry, - pass_config_to_bulk=False, - pass_config_to_single=False, - ) - - @staticmethod - def _load_sidecar_bundle( - media_path: Path, - instance: Optional[str], - config: Dict[str, - Any], - ) -> Tuple[Optional[Path], - Optional[str], - List[str], - List[str]]: - """Load sidecar metadata.""" - return None, None, [], [] - - @staticmethod - def _resolve_file_hash( - result: Any, - media_path: Path, - pipe_obj: models.PipeObject, - fallback_hash: Optional[str], - ) -> Optional[str]: - if pipe_obj.hash and pipe_obj.hash != "unknown": - return pipe_obj.hash - if fallback_hash: - return fallback_hash - - if isinstance(result, dict): - candidate = result.get("hash") - if candidate: - return str(candidate) - - try: - return sha256_file(media_path) - except Exception: - return None - - @staticmethod - def _resolve_media_kind(path: Path) -> str: - return resolve_media_kind_by_extension(path) - - @staticmethod - def _persist_local_metadata( - library_root: Path, - dest_path: Path, - tags: List[str], - url: List[str], - f_hash: Optional[str], - relationships: Any, - duration: Any, - media_kind: str, - ): - pass - - @staticmethod - def _copy_sidecars(source_path: Path, target_path: Path): - possible_sidecars = [ - source_path.with_suffix(source_path.suffix + ".json"), - source_path.with_name(source_path.name + ".tag"), - source_path.with_name(source_path.name + ".metadata"), - source_path.with_name(source_path.name + ".notes"), - ] - for sc in possible_sidecars: - try: - if sc.exists(): - suffix_part = sc.name.replace(source_path.name, "", 1) - dest_sidecar = target_path.parent / f"{target_path.name}{suffix_part}" - dest_sidecar.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(str(sc), dest_sidecar) - except Exception: - pass - - @staticmethod - def _cleanup_after_success(media_path: Path, delete_source: bool): - # Determine whether this is a temporary merge/tracking file which should be - # deleted even when delete_source is False. - is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name - - # If neither explicit delete was requested nor this looks like a temp-merge, - # avoid deleting the source file. - if not delete_source and not is_temp_merge: - return - - # Attempt deletion (best-effort) - try: - media_path.unlink() - Add_File._cleanup_sidecar_files(media_path) - except Exception as exc: - log(f"⚠️ Could not delete file: {exc}", file=sys.stderr) - - @staticmethod - def _cleanup_sidecar_files(media_path: Path): - targets = [ - media_path.parent / (media_path.name + ".metadata"), - media_path.parent / (media_path.name + ".notes"), - media_path.parent / (media_path.name + ".tag"), - ] - for target in targets: - try: - if target.exists(): - target.unlink() - except Exception: - pass - - -# Create and register the cmdlet -CMDLET = Add_File() +from .add_storage import ( + _handle_storage_backend, + _handle_plugin_upload, + _emit_plugin_upload_payload, + _emit_storage_result, + _emit_pipe_object, + _update_pipe_object_destination, + _find_existing_hash_by_urls, + _try_emit_search_file_by_hash, + _try_emit_search_file_by_hashes, + _apply_pending_url_associations, + _apply_pending_tag_associations, + _apply_pending_relationships, + _cleanup_after_success, + _cleanup_sidecar_files, + _copy_sidecars, + _persist_local_metadata, + _stop_live_progress_for_terminal_render, +) diff --git a/cmdlet/file/add_core.py b/cmdlet/file/add_core.py new file mode 100644 index 0000000..f09af8f --- /dev/null +++ b/cmdlet/file/add_core.py @@ -0,0 +1,1000 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional, Sequence, Tuple, List +from pathlib import Path +import sys +import re +from urllib.parse import urlparse + +from SYS import models +from SYS import pipeline as ctx +from SYS.logger import log, debug, debug_panel +from SYS.payload_builders import build_table_result_payload +from SYS.pipeline_progress import PipelineProgress +from SYS.result_publication import overlay_existing_result_table, publish_result_table +from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel +from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS +from PluginCore.backend_registry import BackendRegistry +from .. import _shared as sh + +Cmdlet = sh.Cmdlet +CmdletArg = sh.CmdletArg +parse_cmdlet_args = sh.parse_cmdlet_args +SharedArgs = sh.SharedArgs +extract_tag_from_result = sh.extract_tag_from_result +extract_title_from_result = sh.extract_title_from_result +extract_url_from_result = sh.extract_url_from_result +merge_sequences = sh.merge_sequences +extract_relationships = sh.extract_relationships +extract_duration = sh.extract_duration +coerce_to_pipe_object = sh.coerce_to_pipe_object +collapse_namespace_tags = sh.collapse_namespace_tags +resolve_target_dir = sh.resolve_target_dir +resolve_media_kind_by_extension = sh.resolve_media_kind_by_extension +coerce_to_path = sh.coerce_to_path +build_pipeline_preview = sh.build_pipeline_preview +get_field = sh.get_field + +from SYS.utils import sha256_file, unique_path, sanitize_filename + +SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS +_SCREENSHOT_TIME_SUFFIX_RE = re.compile( + r"^(?P<title>.+?)_(?P<label>(?:\d+h)?(?:\d+m)?\d+s)$", + flags=re.IGNORECASE, +) + +_REMOTE_URL_PREFIXES: tuple[str, ...] = ( + "http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:", "tidal:", "hydrus:", +) + + +class _CommandDependencies: + """Command-scope cache for the backend registry and plugin instances.""" + + def __init__(self, config: Dict[str, Any]) -> None: + self.config = config + self._backend_registry: Optional[BackendRegistry] = None + self._plugins: Dict[str, Any] = {} + + def get_backend_registry(self) -> Optional[BackendRegistry]: + """Lazily initialize and return the command-scope backend registry.""" + if self._backend_registry is None: + try: + self._backend_registry = BackendRegistry(self.config) + except Exception: + self._backend_registry = None + return self._backend_registry + + def get_plugin(self, name: str) -> Optional[Any]: + """Cached plugin lookup by name.""" + from PluginCore.registry import get_plugin + + norm_name = str(name or "").strip().lower() + if not norm_name: + return None + if norm_name in self._plugins: + return self._plugins[norm_name] + + plugin = get_plugin(norm_name, self.config) + self._plugins[norm_name] = plugin + return plugin + + def get_plugin_for_cmdlet(self, name: str, cmdlet_name: str) -> Optional[Any]: + """Cached plugin lookup with explicit cmdlet support check.""" + from PluginCore.registry import get_plugin_for_cmdlet + + norm_name = str(name or "").strip().lower() + if not norm_name: + return None + + cache_key = f"{norm_name}#{cmdlet_name}" + if cache_key in self._plugins: + return self._plugins[cache_key] + + plugin = get_plugin_for_cmdlet(norm_name, cmdlet_name, self.config) + self._plugins[cache_key] = plugin + return plugin + + +class Add_File(Cmdlet): + """Add file into the DB""" + + def __init__(self) -> None: + """Initialize add-file cmdlet.""" + super().__init__( + name="add-file", + summary= + "Ingest a local media file to a configured store or plugin destination.", + usage= + "add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-folder <name>] [-delete]", + arg=[ + CmdletArg( + name="source", + type="string", + required=False, + description="Local file or directory path to ingest or scan.", + ), + SharedArgs.INSTANCE, + SharedArgs.URL, + SharedArgs.PLUGIN, + CmdletArg( + name="delete", + type="flag", + required=False, + description="Delete file after successful upload", + alias="del", + ), + CmdletArg( + name="folder", + description="Folder name override for local plugin exports.", + alias="folder-name", + ), + ], + detail=[ + "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", + "- Store options (use -instance without -plugin):", + " hydrus: Upload to Hydrus database with metadata tagging", + "- Plugin options (use -plugin):", + " local: Copy file to a configured local destination or direct path via -instance", + " local folder exports: use -folder <name> (or -folder-name); piped AllDebrid downloads default to their magnet folder", + " 0x0: Upload to 0x0.st for temporary hosting", + " file.io: Upload to file.io for temporary hosting", + " internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)", + "- Use a positional source path with -instance and -plugin to target a named provider config: add-file C:\\Media\\file.pdf -plugin ftp -instance archive", + ], + examples=[ + 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial', + '@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads', + 'add-file C:\\Media\\report.pdf -plugin ftp -instance archive', + ], + exec=self.run, + ) + self.register() + + @staticmethod + def _uses_legacy_path_flag(args: Sequence[str]) -> bool: + for token in args or []: + lowered = str(token or "").strip().lower() + if lowered in {"-path", "--path", "-p"}: + return True + return False + + @staticmethod + def _legacy_path_flag_message() -> str: + return ( + "add-file no longer supports -path. Pass the source file or directory as a positional argument, " + "and use -plugin local -instance <name|path> for local export." + ) + + @staticmethod + def _normalize_provider_key(value: Optional[Any]) -> Optional[str]: + if value is None: + return None + try: + normalized = str(value).strip().lower() + except Exception: + return None + if not normalized: + return None + if "." in normalized: + normalized = normalized.split(".", 1)[0] + return normalized + + def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: + """Main execution entry point.""" + if Add_File._uses_legacy_path_flag(args): + log(Add_File._legacy_path_flag_message(), file=sys.stderr) + return 1 + + parsed = parse_cmdlet_args(args, self) + progress = PipelineProgress(ctx) + + deps = _CommandDependencies(config) + storage_registry: Optional[BackendRegistry] = None + + source_arg = parsed.get("source") + location = parsed.get("instance") + plugin_instance = parsed.get("instance") + source_url_arg = parsed.get("url") + plugin_name = parsed.get("plugin") + delete_after = parsed.get("delete", False) + folder_name = parsed.get("folder") + local_export_destination: Optional[str] = None + if plugin_name and not plugin_instance and location: + plugin_instance = location + + stage_ctx = ctx.get_stage_context() + is_last_stage = (stage_ctx + is None) or bool(getattr(stage_ctx, + "is_last_stage", + False)) + has_downstream_stage = bool(stage_ctx is not None and not is_last_stage) + + dir_scan_mode = False + dir_scan_results: Optional[List[Dict[str, Any]]] = None + explicit_source_list_results: Optional[List[Dict[str, Any]]] = None + + if source_arg and location and not plugin_name: + try: + source_text = str(source_arg) + except Exception: + source_text = "" + + if "," in source_text: + parts = [p.strip().strip('"') for p in source_text.split(",")] + parts = [p for p in parts if p] + + batch: List[Dict[str, Any]] = [] + for p in parts: + try: + file_path = Path(p) + except Exception: + continue + if not file_path.exists() or not file_path.is_file(): + continue + ext = file_path.suffix.lower() + if ext not in SUPPORTED_MEDIA_EXTENSIONS: + continue + try: + hv = sha256_file(file_path) + except Exception: + continue + try: + size = file_path.stat().st_size + except Exception: + size = 0 + batch.append( + { + "path": file_path, + "name": file_path.name, + "hash": hv, + "size": size, + "ext": ext, + } + ) + + if batch: + explicit_source_list_results = batch + source_arg = None + else: + try: + candidate_dir = Path(str(source_arg)) + if candidate_dir.exists() and candidate_dir.is_dir(): + dir_scan_mode = True + debug( + f"[add-file] Scanning directory for batch add: {candidate_dir}" + ) + dir_scan_results = Add_File._scan_directory_for_files( + candidate_dir + ) + if dir_scan_results: + debug( + f"[add-file] Found {len(dir_scan_results)} supported files in directory" + ) + source_arg = None + except Exception as exc: + debug(f"[add-file] Directory scan failed: {exc}") + + if result is None and not source_arg and not explicit_source_list_results and not dir_scan_results: + try: + if ctx.get_stage_context() is not None: + return 0 + except Exception: + pass + + is_storage_backend_location = False + if location and not plugin_name: + try: + backend_registry_for_lookup = storage_registry or deps.get_backend_registry() + storage_registry = backend_registry_for_lookup + is_storage_backend_location = Add_File._resolve_backend_by_name(backend_registry_for_lookup, str(location)) is not None + except Exception: + is_storage_backend_location = False + + if location and not plugin_name and not is_storage_backend_location: + resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( + location, + config, + deps=deps, + require_explicit=True, + ) + if resolved_local_path: + plugin_name = "local" + plugin_instance = resolved_local_instance or str(location) + location = None + local_export_destination = resolved_local_path + else: + log( + f"Storage backend '{location}' not found. Use -plugin local -instance <name|path> for local export or configure that store backend.", + file=sys.stderr, + ) + return 1 + + normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) + if normalized_plugin_name == "local": + resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( + plugin_instance or location, + config, + deps=deps, + require_explicit=bool(plugin_instance or location), + ) + if not resolved_local_path: + requested_local = str(plugin_instance or location or "").strip() or "<default>" + log( + f"Local destination '{requested_local}' is not configured. Use -plugin local -instance <name|path>.", + file=sys.stderr, + ) + return 1 + plugin_name = "local" + plugin_instance = resolved_local_instance or str(plugin_instance or location or "").strip() or None + location = None + local_export_destination = resolved_local_path + + plugin_storage_backend = None + if plugin_name: + plugin_storage_backend = Add_File._resolve_plugin_storage_backend( + plugin_name, + plugin_instance, + config, + store_instance=storage_registry, + deps=deps, + ) + if plugin_storage_backend and storage_registry is None: + storage_registry = deps.get_backend_registry() + + effective_storage_backend_name = plugin_storage_backend or ( + str(location) if location and is_storage_backend_location else None + ) + + if explicit_source_list_results: + items_to_process = explicit_source_list_results + debug(f"[add-file] Using {len(items_to_process)} files from source list") + elif dir_scan_results: + items_to_process = dir_scan_results + debug(f"[add-file] Using {len(items_to_process)} files from directory scan") + elif source_arg: + items_to_process: List[Any] = [result] + elif isinstance(result, list) and result: + items_to_process = list(result) + else: + items_to_process = [result] + + total_items = len(items_to_process) if isinstance(items_to_process, list) else 0 + processed_items = 0 + try: + ui, _ = progress.ui_and_pipe_index() + if ui is not None and total_items: + preview_items = ( + list(items_to_process) + if isinstance(items_to_process, list) else [items_to_process] + ) + progress.begin_pipe( + total_items=total_items, + items_preview=preview_items, + ) + except Exception: + pass + try: + if total_items: + progress.set_percent(0) + except Exception: + pass + + use_steps = False + steps_started = False + try: + ui, _ = progress.ui_and_pipe_index() + use_steps = (ui is not None) and (len(items_to_process) == 1) + if use_steps: + progress.begin_steps(5) + steps_started = True + except Exception: + use_steps = False + + should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage) + if dir_scan_mode and has_downstream_stage: + debug( + "[add-file] Continuing with directory batch ingest because downstream stages exist" + ) + + if should_present_directory_selector: + try: + from SYS.result_table import Table + from pathlib import Path as _Path + + base_args: List[str] = [] + if plugin_name: + base_args.extend(["-plugin", str(plugin_name)]) + if location: + base_args.extend(["-instance", str(location)]) + if source_url_arg: + base_args.extend(["-url", str(source_url_arg)]) + if bool(delete_after): + base_args.append("-delete") + + table = Table(title="Files in Directory", preserve_order=True) + table.set_table("add-file.directory") + table.set_source_command("add-file", base_args) + + rows: List[Dict[str, Any]] = [] + for file_info in dir_scan_results or []: + p = file_info.get("path") + hp = str(file_info.get("hash") or "") + name = str(file_info.get("name") or "unknown") + try: + clean_title = _Path(name).stem + except Exception: + clean_title = name + ext = str(file_info.get("ext") or "").lstrip(".") + size = file_info.get("size", 0) + + row_item = build_table_result_payload( + title=clean_title, + columns=[ + ("Title", clean_title), + ("Hash", hp), + ("Size", size), + ("Ext", ext), + ], + selection_args=[str(p) if p is not None else ""], + path=str(p) if p is not None else "", + hash=hp, + ) + rows.append(row_item) + table.add_result(row_item) + + ctx.set_current_stage_table(table) + ctx.set_last_result_table( + table, + rows, + subject={ + "table": "add-file.directory" + } + ) + log(f"✓ Found {len(rows)} files. Select with @N (e.g., @1 or @1-3).") + return 0 + except Exception as exc: + debug( + f"[add-file] Failed to display directory scan result table: {exc}" + ) + + collected_payloads: List[Dict[str, Any]] = [] + pending_relationship_pairs: Dict[str, + set[tuple[str, + str]]] = {} + pending_url_associations: Dict[str, + List[tuple[str, + List[str]]]] = {} + pending_tag_associations: Dict[str, + List[tuple[str, + List[str]]]] = {} + successes = 0 + failures = 0 + + live_progress = None + try: + live_progress = ctx.get_live_progress() + except Exception: + live_progress = None + + want_final_search_file = ( + bool(is_last_stage) + and bool(effective_storage_backend_name) + and bool(live_progress) + ) + auto_search_file_after_add = False + + defer_url_association = ( + bool(effective_storage_backend_name) + and len(items_to_process) > 1 + ) + + for idx, item in enumerate(items_to_process, 1): + pipe_obj = coerce_to_pipe_object(item, source_arg) + + if source_url_arg: + try: + from SYS.metadata import normalize_urls + + cli_urls = [u.strip() for u in str(source_url_arg).split(",") if u and u.strip()] + merged_urls: List[str] = [] + + if isinstance(getattr(pipe_obj, "extra", None), dict): + existing_url = pipe_obj.extra.get("url") + if isinstance(existing_url, list): + merged_urls.extend(str(u) for u in existing_url if u) + elif isinstance(existing_url, str) and existing_url.strip(): + merged_urls.append(existing_url.strip()) + else: + pipe_obj.extra = {} + + merged_urls = sh.merge_urls(merged_urls, cli_urls) + if merged_urls: + pipe_obj.extra["url"] = merged_urls + except Exception: + pass + + try: + label = pipe_obj.title + if not label and pipe_obj.path: + try: + label = Path(str(pipe_obj.path)).name + except Exception: + label = pipe_obj.path + if not label: + label = "file" + if total_items: + pending_pct = int(round(((idx - 1) / max(1, total_items)) * 100)) + progress.set_percent(pending_pct) + progress.set_status(f"adding {idx}/{total_items}: {label}") + except Exception: + pass + + temp_dir_to_cleanup: Optional[Path] = None + delete_after_item = delete_after + try: + if use_steps and steps_started: + progress.step("resolving source") + + export_destination = ( + Path(local_export_destination) + if local_export_destination + else Path(location) + if location and not is_storage_backend_location + else None + ) + media_path, file_hash, temp_dir_to_cleanup = self._resolve_source( + item, + source_arg, + pipe_obj, + config, + export_destination=export_destination, + store_instance=storage_registry, + deps=deps, + ) + if not media_path and plugin_name: + media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source( + pipe_obj, config, storage_registry, deps=deps + ) + if not media_path: + failures += 1 + continue + + pipe_obj.path = str(media_path) + + allow_all_files = not bool(effective_storage_backend_name) + if not self._validate_source(media_path, allow_all_extensions=allow_all_files): + failures += 1 + continue + + if use_steps and steps_started: + if not file_hash: + progress.step("hashing file") + progress.step("ingesting file") + + if plugin_name: + if effective_storage_backend_name: + code = self._handle_storage_backend( + item, + media_path, + effective_storage_backend_name, + pipe_obj, + config, + delete_after_item, + collect_payloads=collected_payloads, + collect_relationship_pairs=pending_relationship_pairs, + defer_url_association=defer_url_association, + pending_url_associations=pending_url_associations, + defer_tag_association=defer_url_association, + pending_tag_associations=pending_tag_associations, + suppress_last_stage_overlay=want_final_search_file, + auto_search_file=auto_search_file_after_add, + store_instance=storage_registry, + ) + else: + code = self._handle_plugin_upload( + media_path, + plugin_name, + plugin_instance, + pipe_obj, + config, + delete_after_item, + folder_name=folder_name, + ) + if code == 0: + successes += 1 + else: + failures += 1 + continue + + if location: + try: + backend_registry = storage_registry or deps.get_backend_registry() + resolved_backend = Add_File._resolve_backend_by_name(backend_registry, str(location)) + if resolved_backend is not None: + code = self._handle_storage_backend( + item, + media_path, + location, + pipe_obj, + config, + delete_after_item, + collect_payloads=collected_payloads, + collect_relationship_pairs=pending_relationship_pairs, + defer_url_association=defer_url_association, + pending_url_associations=pending_url_associations, + defer_tag_association=defer_url_association, + pending_tag_associations=pending_tag_associations, + suppress_last_stage_overlay=want_final_search_file, + auto_search_file=auto_search_file_after_add, + store_instance=storage_registry, + ) + else: + log(f"Invalid storage backend: {location}", file=sys.stderr) + code = 1 + except Exception as exc: + debug(f"[add-file] ERROR: Failed to resolve location: {exc}") + log(f"Invalid location: {location}", file=sys.stderr) + failures += 1 + continue + + if code == 0: + successes += 1 + else: + failures += 1 + continue + + log("No destination specified", file=sys.stderr) + failures += 1 + finally: + if temp_dir_to_cleanup is not None: + try: + shutil.rmtree(temp_dir_to_cleanup, ignore_errors=True) + except Exception: + pass + processed_items += 1 + try: + pct = int(round((processed_items / max(1, total_items)) * 100)) + progress.set_percent(pct) + if processed_items >= total_items: + progress.clear_status() + except Exception: + pass + + if pending_url_associations: + try: + Add_File._apply_pending_url_associations( + pending_url_associations, + config, + store_instance=storage_registry + ) + except Exception: + pass + + if pending_tag_associations: + try: + Add_File._apply_pending_tag_associations( + pending_tag_associations, + config, + store_instance=storage_registry + ) + except Exception: + pass + + if want_final_search_file and collected_payloads: + try: + from SYS.rich_display import render_item_details_panel + from SYS.result_table import Table + + Add_File._stop_live_progress_for_terminal_render() + + subject = collected_payloads[0] if len(collected_payloads) == 1 else collected_payloads + from .._shared import display_and_persist_items + display_and_persist_items(collected_payloads, title="Result", subject=subject) + + try: + ctx.set_last_result_items_only(list(collected_payloads)) + except Exception: + pass + except Exception: + pass + + if pending_relationship_pairs: + try: + Add_File._apply_pending_relationships( + pending_relationship_pairs, + config, + store_instance=storage_registry, + deps=deps + ) + except Exception: + pass + + if use_steps and steps_started: + progress.step("finalized") + progress.clear_status() + + if successes > 0: + return 0 + return 1 + + @staticmethod + def validate_preflight_args( + args: Sequence[str], + config: Optional[Dict[str, Any]] = None, + ) -> Optional[str]: + from PluginCore.registry import PLUGIN_REGISTRY + + cfg = config if isinstance(config, dict) else {} + + if Add_File._uses_legacy_path_flag(args): + return f"Pipeline error: {Add_File._legacy_path_flag_message()}" + + try: + parsed = parse_cmdlet_args(args, CMDLET) + except Exception as exc: + return f"Pipeline error: invalid add-file arguments: {exc}" + + deps = _CommandDependencies(cfg) + storage_registry: Optional[BackendRegistry] = None + + location = parsed.get("instance") + plugin_instance = parsed.get("instance") + plugin_name = parsed.get("plugin") + + is_storage_backend_location = False + if location and not plugin_name: + try: + backend_registry_for_lookup = storage_registry or deps.get_backend_registry() + storage_registry = backend_registry_for_lookup + is_storage_backend_location = Add_File._resolve_backend_by_name( + backend_registry_for_lookup, + str(location), + ) is not None + except Exception: + is_storage_backend_location = False + + if location and not plugin_name and not is_storage_backend_location: + resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( + location, + cfg, + deps=deps, + require_explicit=True, + ) + if resolved_local_path: + return None + return ( + f"Pipeline error: storage backend '{location}' not found. " + "Use -plugin local -instance <name|path> for local export or configure that store backend." + ) + + normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) + if normalized_plugin_name: + upload_plugin = deps.get_plugin_for_cmdlet(normalized_plugin_name, "add-file") + if upload_plugin is None: + plugin_info = PLUGIN_REGISTRY.get(normalized_plugin_name) + if plugin_info is not None: + canonical_plugin_name = str(plugin_info.canonical_name or normalized_plugin_name).strip().lower() + if canonical_plugin_name == "loc": + return ( + "Pipeline error: plugin 'loc' does not support add-file. " + "Use -plugin local -instance <name|path> for local export." + ) + if "add-file" not in plugin_info.supported_cmdlets: + return f"Pipeline error: plugin '{canonical_plugin_name}' does not support add-file." + return f"Pipeline error: plugin '{canonical_plugin_name}' is not configured or not available for add-file." + return f"Pipeline error: unknown add-file plugin '{plugin_name}'." + + if normalized_plugin_name == "local": + requested_local = str(plugin_instance or location or "").strip() or "<default>" + resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target( + plugin_instance or location, + cfg, + deps=deps, + require_explicit=bool(plugin_instance or location), + ) + if not resolved_local_path: + return ( + f"Pipeline error: local destination '{requested_local}' is not configured. " + "Use -plugin local -instance <name|path>." + ) + + return None + + @staticmethod + def _resolve_plugin_storage_backend( + plugin_name: Optional[Any], + instance_name: Optional[Any], + config: Dict[str, Any], + *, + store_instance: Optional[Any] = None, + deps: Optional[_CommandDependencies] = None, + ) -> Optional[str]: + plugin_key = Add_File._normalize_provider_key(plugin_name) + if not plugin_key: + return None + + if deps is None: + deps = _CommandDependencies(config) + + file_provider = deps.get_plugin_for_cmdlet(plugin_key, "add-file") + if file_provider is None: + return None + + resolver = getattr(file_provider, "resolve_backend", None) + if not callable(resolver): + return None + + explicit_instance = str(instance_name or "").strip() or None + try: + backend_registry = store_instance if store_instance is not None else deps.get_backend_registry() + except Exception: + backend_registry = None + + try: + resolved_name, backend = resolver( + explicit_instance, + storage=backend_registry, + require_explicit=bool(explicit_instance), + ) + except TypeError: + try: + resolved_name, backend = resolver(explicit_instance) + except Exception: + return None + except Exception: + return None + + if backend is None: + return None + + resolved_text = str(resolved_name or explicit_instance or "").strip() + if not resolved_text: + return None + + checker = getattr(file_provider, "is_backend", None) + if callable(checker): + try: + if not checker(backend, resolved_text): + return None + except Exception: + return None + + return resolved_text + + @staticmethod + def _resolve_local_export_plugin_target( + requested: Optional[Any], + config: Dict[str, Any], + *, + deps: Optional[_CommandDependencies] = None, + require_explicit: bool = False, + ) -> tuple[Optional[str], Optional[str]]: + if deps is None: + deps = _CommandDependencies(config) + + file_provider = deps.get_plugin_for_cmdlet("local", "add-file") + if file_provider is None: + return None, None + + resolver = getattr(file_provider, "resolve_destination", None) + if not callable(resolver): + return None, None + + requested_text = str(requested or "").strip() or None + try: + resolved_name, settings = resolver( + requested_text, + require_explicit=require_explicit, + ) + except TypeError: + try: + resolved_name, settings = resolver(requested_text) + except Exception: + return None, None + except Exception: + return None, None + + path_value = str((settings or {}).get("path") or "").strip() + if not path_value: + return None, None + resolved_text = str(resolved_name or requested_text or "").strip() or None + return resolved_text, path_value + + +# Late-binding: import sub-module functions and attach them as static methods. +# The imports MUST happen after the class is fully defined to avoid circular imports, +# since sub-modules import _CommandDependencies from this module. + +from .add_validation import ( # noqa: E402 + _resolve_source, + _validate_source, + _scan_directory_for_files, + _is_probable_url, + _resolve_backend_by_name, + _download_piped_source, + _maybe_download_plugin_result, + _build_provider_filename, + _maybe_download_backend_file, + _download_remote_backend_url, +) + +from .add_tagging import ( # noqa: E402 + _maybe_apply_florencevision_tags, + _prepare_metadata, + _resolve_file_hash, + _load_sidecar_bundle, + _get_url, + _get_relationships, + _get_duration, + _get_note_text, + _parse_relationship_tag_king_alts, + _parse_relationships_king_alts, + _normalize_hash_candidate, + _resolve_media_kind, +) + +from .add_storage import ( # noqa: E402 + _handle_storage_backend, + _handle_plugin_upload, + _emit_plugin_upload_payload, + _emit_storage_result, + _emit_pipe_object, + _update_pipe_object_destination, + _find_existing_hash_by_urls, + _try_emit_search_file_by_hash, + _try_emit_search_file_by_hashes, + _apply_pending_url_associations, + _apply_pending_tag_associations, + _apply_pending_relationships, + _cleanup_after_success, + _cleanup_sidecar_files, + _copy_sidecars, + _persist_local_metadata, + _stop_live_progress_for_terminal_render, +) + +# Attach sub-module functions as static methods on Add_File +Add_File._resolve_source = staticmethod(_resolve_source) +Add_File._validate_source = staticmethod(_validate_source) +Add_File._scan_directory_for_files = staticmethod(_scan_directory_for_files) +Add_File._is_probable_url = staticmethod(_is_probable_url) +Add_File._resolve_backend_by_name = staticmethod(_resolve_backend_by_name) +Add_File._download_piped_source = staticmethod(_download_piped_source) +Add_File._maybe_download_plugin_result = staticmethod(_maybe_download_plugin_result) +Add_File._build_provider_filename = staticmethod(_build_provider_filename) +Add_File._maybe_download_backend_file = staticmethod(_maybe_download_backend_file) +Add_File._download_remote_backend_url = staticmethod(_download_remote_backend_url) + +Add_File._prepare_metadata = staticmethod(_prepare_metadata) +Add_File._resolve_file_hash = staticmethod(_resolve_file_hash) +Add_File._load_sidecar_bundle = staticmethod(_load_sidecar_bundle) +Add_File._get_url = staticmethod(_get_url) +Add_File._get_relationships = staticmethod(_get_relationships) +Add_File._get_duration = staticmethod(_get_duration) +Add_File._get_note_text = staticmethod(_get_note_text) +Add_File._parse_relationship_tag_king_alts = staticmethod(_parse_relationship_tag_king_alts) +Add_File._parse_relationships_king_alts = staticmethod(_parse_relationships_king_alts) +Add_File._normalize_hash_candidate = staticmethod(_normalize_hash_candidate) +Add_File._resolve_media_kind = staticmethod(_resolve_media_kind) + +Add_File._handle_storage_backend = staticmethod(_handle_storage_backend) +Add_File._handle_plugin_upload = staticmethod(_handle_plugin_upload) +Add_File._emit_plugin_upload_payload = staticmethod(_emit_plugin_upload_payload) +Add_File._emit_storage_result = staticmethod(_emit_storage_result) +Add_File._emit_pipe_object = staticmethod(_emit_pipe_object) +Add_File._update_pipe_object_destination = staticmethod(_update_pipe_object_destination) +Add_File._find_existing_hash_by_urls = staticmethod(_find_existing_hash_by_urls) +Add_File._try_emit_search_file_by_hash = staticmethod(_try_emit_search_file_by_hash) +Add_File._try_emit_search_file_by_hashes = staticmethod(_try_emit_search_file_by_hashes) +Add_File._apply_pending_url_associations = staticmethod(_apply_pending_url_associations) +Add_File._apply_pending_tag_associations = staticmethod(_apply_pending_tag_associations) +Add_File._apply_pending_relationships = staticmethod(_apply_pending_relationships) +Add_File._cleanup_after_success = staticmethod(_cleanup_after_success) +Add_File._cleanup_sidecar_files = staticmethod(_cleanup_sidecar_files) +Add_File._copy_sidecars = staticmethod(_copy_sidecars) +Add_File._persist_local_metadata = staticmethod(_persist_local_metadata) +Add_File._stop_live_progress_for_terminal_render = staticmethod(_stop_live_progress_for_terminal_render) + + +CMDLET = Add_File() diff --git a/cmdlet/file/add_storage.py b/cmdlet/file/add_storage.py new file mode 100644 index 0000000..2526945 --- /dev/null +++ b/cmdlet/file/add_storage.py @@ -0,0 +1,1056 @@ +"""Storage operations, duplicate checking, and emission for add-file.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple +from pathlib import Path +import sys +import shutil +import traceback + +from SYS import models +from SYS import pipeline as ctx +from SYS.logger import log, debug, debug_panel +from SYS.pipeline_progress import PipelineProgress +from SYS.result_publication import publish_result_table, overlay_existing_result_table +from PluginCore.backend_registry import BackendRegistry + +from .. import _shared as sh + +# Import from add_core (safe: add_core defines these before importing this module) +from .add_core import ( + Add_File, + _CommandDependencies, +) + +# Import module-level function from add_tagging +from .add_tagging import _maybe_apply_florencevision_tags + +get_field = sh.get_field + + +def _update_pipe_object_destination( + pipe_obj: models.PipeObject, + *, + hash_value: str, + store: str, + plugin: Optional[str] = None, + path: Optional[str], + tag: List[str], + title: Optional[str], + extra_updates: Optional[Dict[str, Any]] = None, +) -> None: + pipe_obj.hash = hash_value + pipe_obj.store = store + pipe_obj.plugin = plugin + pipe_obj.is_temp = False + pipe_obj.path = path + pipe_obj.tag = tag + if title: + pipe_obj.title = title + if isinstance(pipe_obj.extra, dict): + pipe_obj.extra.update(extra_updates or {}) + else: + pipe_obj.extra = dict(extra_updates or {}) + + +def _stop_live_progress_for_terminal_render() -> None: + try: + live_progress = ctx.get_live_progress() + except Exception: + live_progress = None + + if live_progress is None: + return + + try: + stage_ctx = ctx.get_stage_context() + pipe_idx = getattr(stage_ctx, "pipe_index", None) + if isinstance(pipe_idx, int): + live_progress.finish_pipe(int(pipe_idx), force_complete=True) + except Exception: + pass + + try: + live_progress.stop() + except Exception: + pass + + try: + if hasattr(ctx, "set_live_progress"): + ctx.set_live_progress(None) + except Exception: + pass + + +def _emit_pipe_object(pipe_obj: models.PipeObject) -> None: + payload = pipe_obj.to_dict() + ctx.emit(payload) + ctx.set_current_stage_table(None) + + stage_ctx = ctx.get_stage_context() + is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if not is_last: + return + + try: + _stop_live_progress_for_terminal_render() + from .._shared import display_and_persist_items + + display_and_persist_items([payload], title="Result", subject=payload) + except Exception: + pass + + +def _emit_storage_result( + payload: Dict[str, Any], + *, + overlay: bool = True, + emit: bool = True +) -> None: + """Emit a storage-style result payload. + + - Always emits the dict downstream (when in a pipeline). + - If this is the last stage (or not in a pipeline), prints a search-file-like table + and sets an overlay table/items for @N selection. + """ + if emit: + ctx.emit(payload) + + stage_ctx = ctx.get_stage_context() + is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if not is_last or not overlay: + return + + try: + from SYS.result_table import Table + + table = Table("Result") + table.add_result(payload) + publish_result_table(ctx, table, [payload], subject=payload, overlay=True) + except Exception: + try: + ctx.set_last_result_items_only([payload]) + except Exception: + pass + + +def _try_emit_search_file_by_hash( + *, + instance: str, + hash_value: str, + config: Dict[str, Any] +) -> Optional[List[Any]]: + """Run search-file for a single hash so the final table/payload is consistent.""" + try: + from cmdlet.file.search import CMDLET as search_file_cmdlet + + args = ["-instance", str(instance), f"hash:{str(hash_value)}"] + + prev_ctx = ctx.get_stage_context() + temp_ctx = ctx.PipelineStageContext( + stage_index=0, + total_stages=1, + pipe_index=0, + worker_id=getattr(prev_ctx, "worker_id", None), + ) + ctx.set_stage_context(temp_ctx) + try: + code = search_file_cmdlet.run(None, args, config) + emitted_items = list(getattr(temp_ctx, "emits", []) or []) + finally: + ctx.set_stage_context(prev_ctx) + if code != 0: + return None + + stage_ctx = ctx.get_stage_context() + is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if is_last: + try: + overlay_existing_result_table( + ctx, + subject={ + "store": instance, + "hash": hash_value + }, + ) + except Exception: + pass + + return emitted_items + except Exception as exc: + debug( + f"[add-file] Failed to run search-file after add-file: {type(exc).__name__}: {exc}" + ) + return None + + +def _try_emit_search_file_by_hashes( + *, + instance: str, + hash_values: List[str], + config: Dict[str, Any], + store_instance: Optional[BackendRegistry] = None, +) -> Optional[List[Any]]: + """Run search-file for a list of hashes and promote the table to a display overlay. + + Returns the emitted search-file payload items on success, else None. + """ + hashes = [h for h in (hash_values or []) if isinstance(h, str) and len(h) == 64] + if not instance or not hashes: + return None + + try: + from cmdlet.file.search import CMDLET as search_file_cmdlet + + query = "hash:" + ",".join(hashes) + args = ["-instance", str(instance), "-internal-refresh", query] + debug(f'[add-file] Refresh: search-file -instance {instance} "{query}"') + + prev_ctx = ctx.get_stage_context() + temp_ctx = ctx.PipelineStageContext( + stage_index=0, + total_stages=1, + pipe_index=0, + worker_id=getattr(prev_ctx, "worker_id", None), + ) + ctx.set_stage_context(temp_ctx) + try: + code = search_file_cmdlet.run(None, args, config) + emitted_items = list(getattr(temp_ctx, "emits", []) or []) + finally: + ctx.set_stage_context(prev_ctx) + + if code != 0: + return None + + stage_ctx = ctx.get_stage_context() + is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if is_last: + try: + table = ctx.get_last_result_table() + items = ctx.get_last_result_items() + if table is not None and items: + if len(items) == 1: + try: + from SYS.rich_display import render_item_details_panel + render_item_details_panel(items[0]) + setattr(table, "_rendered_by_cmdlet", True) + except Exception as exc: + debug(f"[add-file] Item details render failed: {exc}") + + publish_result_table( + ctx, + table, + items, + subject={ + "store": instance, + "hash": hashes + }, + overlay=True, + ) + except Exception: + pass + + return emitted_items + except Exception as exc: + debug( + f"[add-file] Failed to run search-file after add-file: {type(exc).__name__}: {exc}" + ) + return None + + +def _find_existing_hash_by_urls( + backend: Any, + urls: Sequence[str], +) -> Optional[str]: + """Best-effort duplicate detection by URL before ingesting file bytes.""" + url_candidates: List[str] = [] + for raw in urls or []: + text = str(raw or "").strip() + if not text or not Add_File._is_probable_url(text): + continue + if text not in url_candidates: + url_candidates.append(text) + + if not url_candidates: + return None + + lookup_exact = getattr(backend, "find_hashes_by_url", None) + if callable(lookup_exact): + for candidate_url in url_candidates: + try: + hashes = lookup_exact(candidate_url) or [] + except Exception: + continue + if not isinstance(hashes, (list, tuple, set)): + continue + for item in hashes: + normalized = Add_File._normalize_hash_candidate(item) + if normalized: + return normalized + + searcher = getattr(backend, "search", None) + if callable(searcher): + for candidate_url in url_candidates: + try: + hits = searcher(f"url:{candidate_url}", limit=1, minimal=True) or [] + except Exception: + continue + if not isinstance(hits, list) or not hits: + continue + hit = hits[0] + for key in ("hash", "file_hash", "sha256"): + normalized = Add_File._normalize_hash_candidate(get_field(hit, key)) + if normalized: + return normalized + + return None + + +def _emit_plugin_upload_payload( + upload_payload: Dict[str, Any], + plugin_name: str, + instance_name: Optional[str], + pipe_obj: models.PipeObject, + media_path: Path, + delete_after: bool, +) -> int: + payload = dict(upload_payload or {}) + extra_updates: Dict[str, Any] = {} + raw_extra = payload.get("extra") + if isinstance(raw_extra, dict): + extra_updates.update(raw_extra) + + if plugin_name: + extra_updates.setdefault("plugin", plugin_name) + if instance_name: + extra_updates.setdefault("instance", instance_name) + + raw_urls = payload.get("url") + if isinstance(raw_urls, str): + url_values = [raw_urls.strip()] if raw_urls.strip() else [] + extra_updates["url"] = url_values + elif isinstance(raw_urls, (list, tuple, set)): + url_values = [str(item).strip() for item in raw_urls if str(item).strip()] + extra_updates["url"] = url_values + + relationships = payload.get("relationships") + if relationships: + try: + pipe_obj.relationships = relationships + except Exception: + pass + + tags = payload.get("tag") + if isinstance(tags, list): + tag_values = [str(tag) for tag in tags] + else: + tag_values = list(pipe_obj.tag or []) + + title_value = str(payload.get("title") or pipe_obj.title or media_path.name).strip() or media_path.name + path_value = str(payload.get("path") or pipe_obj.path or media_path).strip() + hash_value = str( + payload.get("hash") + or payload.get("file_hash") + or getattr(pipe_obj, "hash", None) + or "unknown" + ).strip() or "unknown" + store_value = str(payload.get("store") or "").strip() + plugin_value = payload.get("plugin") + if plugin_value is None and plugin_name: + plugin_value = plugin_name + + _update_pipe_object_destination( + pipe_obj, + hash_value=hash_value, + store=store_value, + plugin=str(plugin_value) if plugin_value else None, + path=path_value, + tag=tag_values, + title=title_value, + extra_updates=extra_updates, + ) + _emit_pipe_object(pipe_obj) + + _cleanup_after_success(media_path, delete_source=delete_after) + return 0 + + +def _handle_plugin_upload( + media_path: Path, + plugin_name: str, + instance_name: Optional[str], + pipe_obj: models.PipeObject, + config: Dict[str, Any], + delete_after: bool, + folder_name: Optional[str] = None, +) -> int: + """Handle uploading via an add-file plugin (e.g. 0x0).""" + from PluginCore.registry import ( + PLUGIN_REGISTRY, + get_plugin_for_cmdlet, + list_plugins_for_cmdlet, + ) + + try: + file_provider = get_plugin_for_cmdlet(plugin_name, "add-file", config) + if not file_provider: + available_map = list_plugins_for_cmdlet("add-file", config) + available_add_plugins = [n for n, e in available_map.items() if e] + requested_plugin_info = PLUGIN_REGISTRY.get(plugin_name) + + if requested_plugin_info is not None and "add-file" in requested_plugin_info.supported_cmdlets: + from SYS.rich_display import show_plugin_config_panel + show_plugin_config_panel([requested_plugin_info.canonical_name]) + else: + log(f"Add-file plugin '{plugin_name}' is not available or does not support add-file", file=sys.stderr) + + if available_add_plugins: + from SYS.rich_display import show_available_plugins_panel + show_available_plugins_panel(sorted(available_add_plugins)) + return 1 + + upload_kwargs: Dict[str, Any] = { + "pipe_obj": pipe_obj, + "instance": instance_name, + } + pipeline_progress = PipelineProgress(ctx) + normalized_plugin_name = Add_File._normalize_provider_key(plugin_name) + f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None) + if normalized_plugin_name == "local": + result = None + tags, urls, title, f_hash = Add_File._prepare_metadata(result, media_path, pipe_obj, config) + relationships = Add_File._get_relationships(result, pipe_obj) + direct_export_download = False + try: + if isinstance(pipe_obj.extra, dict): + direct_export_download = bool(pipe_obj.extra.pop("_direct_export_download", False)) + except Exception: + direct_export_download = False + + upload_kwargs.update( + { + "title": title, + "tags": tags, + "urls": urls, + "hash_value": f_hash, + "relationships": relationships, + "direct_export_download": direct_export_download, + "pipeline_progress": pipeline_progress, + } + ) + if folder_name is not None and str(folder_name).strip(): + upload_kwargs["folder_name"] = str(folder_name).strip() + + upload_result = file_provider.upload( + str(media_path), + **upload_kwargs, + ) + + duplicate_upload = False + duplicate_rule = "" + duplicate_target = "" + try: + if isinstance(getattr(pipe_obj, "extra", None), dict): + duplicate_upload = bool(pipe_obj.extra.get("upload_duplicate")) + duplicate_rule = str(pipe_obj.extra.get("upload_duplicate_rule") or "").strip() + duplicate_target = str(pipe_obj.extra.get("upload_duplicate_target") or "").strip() + except Exception: + duplicate_upload = False + duplicate_rule = "" + duplicate_target = "" + + except Exception as exc: + log(f"Upload failed: {exc}", file=sys.stderr) + return 1 + + if isinstance(upload_result, dict): + return _emit_plugin_upload_payload( + upload_result, + plugin_name, + instance_name, + pipe_obj, + media_path, + delete_after, + ) + + hoster_url = str(upload_result or "").strip() + + extra_updates: Dict[str, Any] = { + "plugin": plugin_name, + "instance": instance_name, + "plugin_url": hoster_url, + } + if isinstance(pipe_obj.extra, dict): + existing_known = list(pipe_obj.extra.get("url") or []) + if hoster_url and hoster_url not in existing_known: + existing_known.append(hoster_url) + extra_updates["url"] = existing_known + + file_path = pipe_obj.path or (str(media_path) if media_path else None) or "" + _update_pipe_object_destination( + pipe_obj, + hash_value=f_hash or "unknown", + store="", + plugin=plugin_name or None, + path=file_path, + tag=pipe_obj.tag, + title=pipe_obj.title or (media_path.name if media_path else None), + extra_updates=extra_updates, + ) + _emit_pipe_object(pipe_obj) + + _cleanup_after_success(media_path, delete_source=delete_after) + return 0 + + +def _handle_storage_backend( + result: Any, + media_path: Path, + backend_name: str, + pipe_obj: models.PipeObject, + config: Dict[str, Any], + delete_after: bool, + *, + collect_payloads: Optional[List[Dict[str, Any]]] = None, + collect_relationship_pairs: Optional[Dict[str, set[tuple[str, str]]]] = None, + defer_url_association: bool = False, + pending_url_associations: Optional[Dict[str, List[tuple[str, List[str]]]]] = None, + defer_tag_association: bool = False, + pending_tag_associations: Optional[Dict[str, List[tuple[str, List[str]]]]] = None, + suppress_last_stage_overlay: bool = False, + auto_search_file: bool = True, + store_instance: Optional[BackendRegistry] = None, +) -> int: + """Handle uploading to a registered storage backend (e.g., 'test' folder store, 'hydrus', etc.).""" + pipeline_progress = PipelineProgress(ctx) + + def _set_status(text: str) -> None: + try: + pipeline_progress.set_status(f"{backend_name}: {text}") + except Exception: + pass + + def _clear_status() -> None: + try: + pipeline_progress.clear_status() + except Exception: + pass + + delete_after_effective = bool(delete_after) + if not delete_after_effective: + try: + if (str(backend_name or "").strip().lower() != "temp" + and getattr(pipe_obj, "is_temp", False) + and getattr(pipe_obj, "action", None) == "cmdlet:download-media"): + from SYS.config import resolve_output_dir + + temp_dir = resolve_output_dir(config) + try: + if media_path.resolve().is_relative_to( + temp_dir.expanduser().resolve()): + delete_after_effective = True + debug( + f"[add-file] Auto-delete temp source after ingest: {media_path}" + ) + except Exception: + pass + except Exception: + pass + + try: + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) + backend, backend_registry, backend_exc = sh.get_preferred_store_backend( + config, + backend_name, + store_registry=backend_registry, + suppress_debug=True, + ) + if backend is None: + raise backend_exc or KeyError(f"Unknown store backend: {backend_name}") + + is_remote_backend = getattr(backend, "is_remote", False) + prefer_defer_tags = getattr(backend, "prefer_defer_tags", False) + supports_url_association = bool(getattr(backend, "supports_url_association", False)) + supports_note_association = bool(getattr(backend, "supports_note_association", False)) + supports_relationship_association = bool(getattr(backend, "supports_relationship_association", False)) + + tags, url, title, f_hash = Add_File._prepare_metadata( + result, media_path, pipe_obj, config + ) + + try: + from SYS.metadata import normalize_urls + + source_store = None + source_hash = None + if isinstance(result, dict): + source_store = result.get("store") + source_hash = result.get("hash") + if not source_store: + source_store = getattr(pipe_obj, "store", None) + if not source_hash: + source_hash = getattr(pipe_obj, "hash", None) + if (not source_hash) and isinstance(pipe_obj.extra, dict): + source_hash = pipe_obj.extra.get("hash") + + source_store = str(source_store or "").strip() + source_hash = str(source_hash or "").strip().lower() + if (source_store and source_hash and len(source_hash) == 64 + and source_store.lower() != str(backend_name or "").strip().lower()): + source_backend = None + try: + store = backend_registry + if source_store in store.list_backends(): + source_backend = store[source_store] + except Exception: + source_backend = None + + if source_backend is not None: + try: + src_urls = normalize_urls( + source_backend.get_url(source_hash) or [] + ) + except Exception: + src_urls = [] + + try: + dst_urls = normalize_urls(url or []) + except Exception: + dst_urls = [] + + merged: list[str] = [] + seen: set[str] = set() + for u in list(dst_urls or []) + list(src_urls or []): + if not u: + continue + if u in seen: + continue + seen.add(u) + merged.append(u) + url = merged + except Exception: + pass + + if collect_relationship_pairs is not None and supports_relationship_association: + rels = Add_File._get_relationships(result, pipe_obj) + if isinstance(rels, dict) and rels: + king_hash, alt_hashes = Add_File._parse_relationships_king_alts(rels) + if king_hash and alt_hashes: + bucket = collect_relationship_pairs.setdefault( + str(backend_name), + set() + ) + for alt_hash in alt_hashes: + if alt_hash and alt_hash != king_hash: + bucket.add((alt_hash, king_hash)) + + if isinstance(tags, list) and tags: + tags = [ + t for t in tags if not ( + isinstance(t, str) + and t.strip().lower().startswith("relationship:") + ) + ] + + try: + tags = _maybe_apply_florencevision_tags(media_path, list(tags or []), config, pipe_obj=pipe_obj) + pipe_obj.tag = list(tags or []) + except Exception as exc: + log(f"[add-file] FlorenceVision tagging error: {exc}", file=sys.stderr) + return 1 + + upload_tags = tags + if prefer_defer_tags and upload_tags: + upload_tags = [] + try: + debug_panel( + "add-file store", + [ + ("backend", backend_name), + ("path", media_path), + ("title", title), + ("hash_hint", f_hash[:12] if f_hash else "N/A"), + ("defer_tags", bool(prefer_defer_tags and tags)), + ], + border_style="yellow", + ) + except Exception: + pass + + duplicate_hash = Add_File._find_existing_hash_by_urls(backend, url) + if duplicate_hash: + debug( + f"[add-file] URL duplicate detected in '{backend_name}', skipping upload and reusing hash {duplicate_hash[:12]}..." + ) + file_identifier = duplicate_hash + else: + file_identifier = backend.add_file( + media_path, + title=title, + tag=upload_tags, + url=[] if ((defer_url_association and url) or (not supports_url_association)) else url, + file_hash=f_hash, + pipeline_progress=pipeline_progress, + transfer_label=title or media_path.name, + ) + + stored_path: Optional[str] = None + try: + if not is_remote_backend: + maybe_path = backend.get_file(file_identifier) + if isinstance(maybe_path, Path): + stored_path = str(maybe_path) + elif isinstance(maybe_path, str) and maybe_path: + stored_path = maybe_path + except Exception: + stored_path = None + + if isinstance(file_identifier, str) and len(file_identifier) == 64: + chosen_hash = file_identifier + else: + chosen_hash = f_hash or (str(file_identifier) if file_identifier is not None else "unknown") + + _update_pipe_object_destination( + pipe_obj, + hash_value=chosen_hash, + store=backend_name, + path=stored_path, + tag=tags, + title=title or pipe_obj.title or media_path.name, + extra_updates={ + "url": url, + }, + ) + + resolved_hash = chosen_hash + + if prefer_defer_tags and tags: + if defer_tag_association and pending_tag_associations is not None: + try: + pending_tag_associations.setdefault(str(backend_name), []).append((str(resolved_hash), list(tags))) + except Exception: + pass + else: + try: + adder = getattr(backend, "add_tag", None) + if callable(adder): + _set_status("applying deferred tags") + adder(resolved_hash, list(tags)) + except Exception as exc: + log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr) + + if url and supports_url_association: + if defer_url_association and pending_url_associations is not None: + try: + pending_url_associations.setdefault( + str(backend_name), + [] + ).append((str(resolved_hash), list(url))) + except Exception: + pass + else: + try: + is_folder_backend = getattr(backend, "STORE_TYPE", "") == "folder" + if not is_folder_backend: + _set_status("associating urls") + backend.add_url(resolved_hash, list(url)) + except Exception: + pass + + def _write_note(note_name: str, note_text: Optional[str]) -> None: + if not note_text or not supports_note_association: + return + try: + setter = getattr(backend, "set_note", None) + if callable(setter): + _set_status(f"writing {note_name} note") + setter(resolved_hash, note_name, note_text) + except Exception as exc: + debug_panel( + "add-file note write failed", + [ + ("store", backend_name), + ("hash", resolved_hash), + ("note", note_name), + ("error", exc), + ], + border_style="yellow", + ) + + _write_note("sub", Add_File._get_note_text(result, pipe_obj, "sub")) + _write_note("lyric", Add_File._get_note_text(result, pipe_obj, "lyric")) + _write_note("chapters", Add_File._get_note_text(result, pipe_obj, "chapters")) + _write_note("caption", Add_File._get_note_text(result, pipe_obj, "caption")) + + meta: Dict[str, Any] = {} + try: + is_folder_backend_meta = getattr(backend, "STORE_TYPE", "") == "folder" + if not is_folder_backend_meta: + _set_status("loading stored metadata") + meta = backend.get_metadata(resolved_hash) or {} + except Exception: + meta = {} + + size_bytes: Optional[int] = None + for key in ("size_bytes", "size", "filesize", "file_size"): + try: + raw_size = meta.get(key) + if raw_size is not None: + size_bytes = int(raw_size) + break + except Exception: + pass + if size_bytes is None: + try: + size_bytes = int(media_path.stat().st_size) + except Exception: + size_bytes = None + + title_out = ( + meta.get("title") or title or pipe_obj.title or media_path.stem + or media_path.name + ) + ext_out = meta.get("ext") or media_path.suffix.lstrip(".") + + payload: Dict[str, Any] = { + "title": title_out, + "ext": str(ext_out or ""), + "size_bytes": size_bytes, + "store": backend_name, + "hash": resolved_hash, + "path": stored_path, + "tag": list(tags or []), + "url": list(url or []), + } + if collect_payloads is not None: + try: + collect_payloads.append(payload) + except Exception: + pass + + if auto_search_file and resolved_hash and resolved_hash != "unknown": + _emit_storage_result( + payload, + overlay=not suppress_last_stage_overlay, + emit=False + ) + + refreshed_items = _try_emit_search_file_by_hash( + instance=backend_name, + hash_value=resolved_hash, + config=config, + ) + if refreshed_items: + for emitted in refreshed_items: + ctx.emit(emitted) + else: + ctx.emit(payload) + else: + _emit_storage_result( + payload, + overlay=not suppress_last_stage_overlay, + emit=True + ) + + _cleanup_after_success( + media_path, + delete_source=delete_after_effective + ) + _clear_status() + return 0 + + except Exception as exc: + _clear_status() + log( + f"❌ Failed to add file to backend '{backend_name}': {exc}", + file=sys.stderr + ) + traceback.print_exc(file=sys.stderr) + return 1 + + +def _apply_pending_url_associations( + pending: Dict[str, List[tuple[str, List[str]]]], + config: Dict[str, Any], + store_instance: Optional[BackendRegistry] = None, +) -> None: + """Apply deferred URL associations in bulk, grouped per backend.""" + + try: + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) + except Exception: + return + + for backend_name, pairs in (pending or {}).items(): + if not pairs: + continue + try: + backend, backend_registry, _exc = sh.get_store_backend( + config, + backend_name, + store_registry=backend_registry, + ) + if backend is None: + continue + + if not bool(getattr(backend, "supports_url_association", False)): + continue + + items = sh.coalesce_hash_value_pairs(pairs) + if not items: + continue + + bulk = getattr(backend, "add_url_bulk", None) + if callable(bulk): + try: + bulk(items) + continue + except Exception: + pass + + single = getattr(backend, "add_url", None) + if callable(single): + for h, u in items: + try: + single(h, u) + except Exception: + continue + except Exception: + continue + + +def _apply_pending_tag_associations( + pending: Dict[str, List[tuple[str, List[str]]]], + config: Dict[str, Any], + store_instance: Optional[BackendRegistry] = None, +) -> None: + """Apply deferred tag associations in bulk, grouped per backend.""" + + try: + backend_registry = store_instance if store_instance is not None else BackendRegistry(config) + except Exception: + return + + sh.run_store_hash_value_batches( + config, + pending or {}, + bulk_method_name="add_tags_bulk", + single_method_name="add_tag", + store_registry=backend_registry, + pass_config_to_bulk=False, + pass_config_to_single=False, + ) + + +def _apply_pending_relationships( + pending: Dict[str, set[tuple[str, str]]], + config: Dict[str, Any], + store_instance: Optional[BackendRegistry] = None, + deps: Optional[_CommandDependencies] = None, +) -> None: + """Persist relationships to backends that support relationships. + + This delegates to an optional backend method: `set_relationship(alt, king, kind)`. + """ + if not pending: + return + + if deps is None: + deps = _CommandDependencies(config) + + try: + backend_registry = store_instance if store_instance is not None else deps.get_backend_registry() + except Exception: + return + + for backend_name, pairs in pending.items(): + if not pairs: + continue + + try: + backend = backend_registry[str(backend_name)] + except Exception: + continue + + if not bool(getattr(backend, "supports_relationship_association", False)): + continue + + setter = getattr(backend, "set_relationship", None) + if not callable(setter): + continue + + processed_pairs: set[tuple[str, str]] = set() + for alt_hash, king_hash in sorted(pairs): + if not alt_hash or not king_hash or alt_hash == king_hash: + continue + if (alt_hash, king_hash) in processed_pairs: + continue + alt_norm = str(alt_hash).strip().lower() + king_norm = str(king_hash).strip().lower() + if len(alt_norm) != 64 or len(king_norm) != 64: + continue + try: + setter(alt_norm, king_norm, "alt") + processed_pairs.add((alt_hash, king_hash)) + except Exception: + continue + + +def _cleanup_after_success(media_path: Path, delete_source: bool): + is_temp_merge = "(merged)" in media_path.name or ".dlhx_" in media_path.name + + if not delete_source and not is_temp_merge: + return + + try: + media_path.unlink() + _cleanup_sidecar_files(media_path) + except Exception as exc: + log(f"⚠️ Could not delete file: {exc}", file=sys.stderr) + + +def _cleanup_sidecar_files(media_path: Path): + targets = [ + media_path.parent / (media_path.name + ".metadata"), + media_path.parent / (media_path.name + ".notes"), + media_path.parent / (media_path.name + ".tag"), + ] + for target in targets: + try: + if target.exists(): + target.unlink() + except Exception: + pass + + +def _copy_sidecars(source_path: Path, target_path: Path): + possible_sidecars = [ + source_path.with_suffix(source_path.suffix + ".json"), + source_path.with_name(source_path.name + ".tag"), + source_path.with_name(source_path.name + ".metadata"), + source_path.with_name(source_path.name + ".notes"), + ] + for sc in possible_sidecars: + try: + if sc.exists(): + suffix_part = sc.name.replace(source_path.name, "", 1) + dest_sidecar = target_path.parent / f"{target_path.name}{suffix_part}" + dest_sidecar.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(sc), dest_sidecar) + except Exception: + pass + + +def _persist_local_metadata( + library_root: Path, + dest_path: Path, + tags: List[str], + url: List[str], + f_hash: Optional[str], + relationships: Any, + duration: Any, + media_kind: str, +): + pass diff --git a/cmdlet/file/add_tagging.py b/cmdlet/file/add_tagging.py new file mode 100644 index 0000000..e6c09ee --- /dev/null +++ b/cmdlet/file/add_tagging.py @@ -0,0 +1,562 @@ +"""Tag resolution, FlorenceVision integration, and metadata extraction for add-file.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Tuple +from pathlib import Path +import sys +import re + +from SYS import models +from SYS.logger import log, debug + +from .. import _shared as sh + +# Import from add_core (safe: add_core defines these before importing this module) +from .add_core import ( + Add_File, + _SCREENSHOT_TIME_SUFFIX_RE, + _REMOTE_URL_PREFIXES, +) + +extract_tag_from_result = sh.extract_tag_from_result +extract_title_from_result = sh.extract_title_from_result +extract_url_from_result = sh.extract_url_from_result +merge_sequences = sh.merge_sequences +extract_relationships = sh.extract_relationships +extract_duration = sh.extract_duration +collapse_namespace_tags = sh.collapse_namespace_tags +resolve_media_kind_by_extension = sh.resolve_media_kind_by_extension +get_field = sh.get_field + + +def _maybe_apply_florencevision_tags( + media_path: Path, + tags: List[str], + config: Dict[str, Any], + pipe_obj: Optional[models.PipeObject] = None, +) -> List[str]: + """Optionally auto-tag images using the FlorenceVision plugin helper. + + Controlled via config: + [plugin=florencevision] + enabled=true + strict=false + + If strict=false (default), failures log a warning and return the original tags. + If strict=true, failures raise to abort the ingest. + """ + strict = False + try: + plugin_block = (config or {}).get("plugin") + fv_block = plugin_block.get("florencevision") if isinstance(plugin_block, dict) else None + enabled = False + if isinstance(fv_block, dict): + enabled = bool(fv_block.get("enabled")) + strict = bool(fv_block.get("strict")) + if not enabled: + return tags + + from plugins.florencevision import FlorenceVisionTool + + cfg_for_tool: Dict[str, Any] = config + try: + action = str(getattr(pipe_obj, "action", "") or "") if pipe_obj is not None else "" + cmdlet_name = "" + if action.lower().startswith("cmdlet:"): + cmdlet_name = action.split(":", 1)[1].strip().lower() + if cmdlet_name in {"screen-shot", "screen_shot", "screenshot"}: + plugin_block2 = dict((config or {}).get("plugin") or {}) + fv_block2 = dict(plugin_block2.get("florencevision") or {}) + fv_block2["task"] = "ocr" + plugin_block2["florencevision"] = fv_block2 + cfg_for_tool = dict(config or {}) + cfg_for_tool["plugin"] = plugin_block2 + except Exception: + cfg_for_tool = config + + fv = FlorenceVisionTool(cfg_for_tool) + if not fv.enabled() or not fv.applicable_path(media_path): + return tags + + auto_tags = fv.tags_for_file(media_path) + + try: + caption_text = getattr(fv, "last_caption", None) + if caption_text and pipe_obj is not None: + if not isinstance(pipe_obj.extra, dict): + pipe_obj.extra = {} + notes = pipe_obj.extra.get("notes") + if not isinstance(notes, dict): + notes = {} + notes.setdefault("caption", caption_text) + pipe_obj.extra["notes"] = notes + except Exception: + pass + + if not auto_tags: + return tags + + merged = merge_sequences(tags or [], auto_tags, case_sensitive=False) + debug(f"[add-file] FlorenceVision added {len(auto_tags)} tag(s)") + return merged + except Exception as exc: + strict2 = False + try: + tool_block = (config or {}).get("tool") + fv_block = tool_block.get("florencevision") if isinstance(tool_block, dict) else None + strict2 = bool(fv_block.get("strict")) if isinstance(fv_block, dict) else False + except Exception: + strict2 = False + + if strict or strict2: + raise + log(f"[add-file] Warning: FlorenceVision tagging failed: {exc}", file=sys.stderr) + return tags + + +def _normalize_hash_candidate(value: Any) -> str: + text = str(value or "").strip().lower() + if len(text) != 64: + return "" + if any(ch not in "0123456789abcdef" for ch in text): + return "" + return text + + +def _parse_relationship_tag_king_alts( + tag_value: str +) -> tuple[Optional[str], List[str]]: + """Parse a relationship tag into (king_hash, alt_hashes). + + Supported formats: + - New: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH> + - Old: relationship: hash(king)<KING_HASH>,hash(alt)<ALT_HASH>... + relationship: hash(king)KING,hash(alt)ALT + + For the local DB we treat the first hash listed as the king. + """ + if not isinstance(tag_value, str): + return None, [] + + raw = tag_value.strip() + if not raw: + return None, [] + + rhs = raw + if ":" in raw: + prefix, rest = raw.split(":", 1) + if prefix.strip().lower() == "relationship": + rhs = rest.strip() + + typed = re.findall(r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?", rhs) + if typed: + king: Optional[str] = None + alts: List[str] = [] + for rel_type, h in typed: + h_norm = str(h).strip().lower() + if rel_type.strip().lower() == "king": + king = h_norm + elif rel_type.strip().lower() in {"alt", "related"}: + alts.append(h_norm) + if not king: + all_hashes = [str(h).strip().lower() for _, h in typed] + king = all_hashes[0] if all_hashes else None + alts = [h for h in all_hashes[1:] if h] + seen: set[str] = set() + alts = [ + h for h in alts + if h and len(h) == 64 and not (h in seen or seen.add(h)) + ] + if king and len(king) == 64: + return king, [h for h in alts if h != king] + return None, [] + + hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", rhs) + hashes = [h.strip().lower() for h in hashes if isinstance(h, str)] + if not hashes: + return None, [] + king = hashes[0] + alts = hashes[1:] + seen2: set[str] = set() + alts = [ + h for h in alts if h and len(h) == 64 and not (h in seen2 or seen2.add(h)) + ] + return king, [h for h in alts if h != king] + + +def _parse_relationships_king_alts( + relationships: Dict[str, Any], +) -> tuple[Optional[str], List[str]]: + """Parse a PipeObject.relationships dict into (king_hash, alt_hashes). + + Supported shapes: + - {"king": [KING], "alt": [ALT1, ALT2]} + - {"king": KING, "alt": ALT} (strings) + - Also treats "related" hashes as alts for persistence purposes. + """ + if not isinstance(relationships, dict) or not relationships: + return None, [] + + def _first_hash(val: Any) -> Optional[str]: + if isinstance(val, str): + h = val.strip().lower() + return h if len(h) == 64 else None + if isinstance(val, list): + for item in val: + if isinstance(item, str): + h = item.strip().lower() + if len(h) == 64: + return h + return None + + def _many_hashes(val: Any) -> List[str]: + out: List[str] = [] + if isinstance(val, str): + h = val.strip().lower() + if len(h) == 64: + out.append(h) + elif isinstance(val, list): + for item in val: + if isinstance(item, str): + h = item.strip().lower() + if len(h) == 64: + out.append(h) + return out + + king = _first_hash(relationships.get("king")) + if not king: + return None, [] + + alts = _many_hashes(relationships.get("alt")) + alts.extend(_many_hashes(relationships.get("related"))) + + seen: set[str] = set() + alts = [h for h in alts if h and h != king and not (h in seen or seen.add(h))] + return king, alts + + +def _get_url(result: Any, pipe_obj: models.PipeObject) -> List[str]: + """Extract valid URLs from pipe object or result dict.""" + from SYS.metadata import normalize_urls + + candidates: List[str] = [] + + if pipe_obj.url: + candidates.append(pipe_obj.url) + if pipe_obj.source_url: + candidates.append(pipe_obj.source_url) + + if isinstance(pipe_obj.extra, dict): + u = pipe_obj.extra.get("url") + if isinstance(u, list): + candidates.extend(str(x) for x in u if x) + elif isinstance(u, str): + candidates.append(u) + + raw_from_result = extract_url_from_result(result) + if raw_from_result: + candidates.extend(raw_from_result) + + normalized = normalize_urls(candidates) + return [u for u in normalized if Add_File._is_probable_url(u)] + + +def _get_relationships(result: Any, pipe_obj: models.PipeObject) -> Optional[Dict[str, Any]]: + try: + rels = pipe_obj.get_relationships() + if rels: + return rels + except Exception: + pass + if isinstance(result, dict) and result.get("relationships"): + return result.get("relationships") + try: + return extract_relationships(result) + except Exception: + return None + + +def _get_duration(result: Any, pipe_obj: models.PipeObject) -> Optional[float]: + + def _parse_duration(value: Any) -> Optional[float]: + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) if value > 0 else None + if isinstance(value, str): + s = value.strip() + if not s: + return None + try: + candidate = float(s) + return candidate if candidate > 0 else None + except ValueError: + pass + if ":" in s: + parts = [p.strip() for p in s.split(":") if p.strip()] + if len(parts) in {2, 3} and all(p.isdigit() for p in parts): + nums = [int(p) for p in parts] + if len(nums) == 2: + minutes, seconds = nums + return float(minutes * 60 + seconds) + hours, minutes, seconds = nums + return float(hours * 3600 + minutes * 60 + seconds) + return None + + parsed = _parse_duration(getattr(pipe_obj, "duration", None)) + if parsed is not None: + return parsed + try: + return _parse_duration(extract_duration(result)) + except Exception: + return None + + +def _get_note_text(result: Any, pipe_obj: models.PipeObject, note_name: str) -> Optional[str]: + """Extract a named note text from a piped item. + + Supports: + - pipe_obj.extra["notes"][note_name] + - result["notes"][note_name] for dict results + - pipe_obj.extra[note_name] / result[note_name] as fallback + """ + + def _normalize(val: Any) -> Optional[str]: + if val is None: + return None + if isinstance(val, bytes): + try: + val = val.decode("utf-8", errors="ignore") + except Exception: + val = str(val) + if isinstance(val, str): + text = val.strip() + return text if text else None + try: + text = str(val).strip() + return text if text else None + except Exception: + return None + + note_key = str(note_name or "").strip() + if not note_key: + return None + + try: + if isinstance(pipe_obj.extra, dict): + notes_val = pipe_obj.extra.get("notes") + if isinstance(notes_val, dict) and note_key in notes_val: + return _normalize(notes_val.get(note_key)) + if note_key in pipe_obj.extra: + return _normalize(pipe_obj.extra.get(note_key)) + except Exception: + pass + + if isinstance(result, dict): + try: + notes_val = result.get("notes") + if isinstance(notes_val, dict) and note_key in notes_val: + return _normalize(notes_val.get(note_key)) + if note_key in result: + return _normalize(result.get(note_key)) + except Exception: + pass + + return None + + +def _load_sidecar_bundle( + media_path: Path, + instance: Optional[str], + config: Dict[str, Any], +) -> Tuple[Optional[Path], Optional[str], List[str], List[str]]: + """Load sidecar metadata (placeholder — overridden by active plugins).""" + return None, None, [], [] + + +def _resolve_file_hash( + result: Any, + media_path: Path, + pipe_obj: models.PipeObject, + fallback_hash: Optional[str], +) -> Optional[str]: + from SYS.utils import sha256_file + + if pipe_obj.hash and pipe_obj.hash != "unknown": + return pipe_obj.hash + if fallback_hash: + return fallback_hash + + if isinstance(result, dict): + candidate = result.get("hash") + if candidate: + return str(candidate) + + try: + return sha256_file(media_path) + except Exception: + return None + + +def _resolve_media_kind(path: Path) -> str: + return resolve_media_kind_by_extension(path) + + +def _prepare_metadata( + result: Any, + media_path: Path, + pipe_obj: models.PipeObject, + config: Dict[str, Any], +) -> Tuple[List[str], List[str], Optional[str], Optional[str]]: + """ + Prepare tags, url, and title for the file. + Returns (tags, url, preferred_title, file_hash) + """ + tags_from_result = list(pipe_obj.tag or []) + if not tags_from_result: + try: + tags_from_result = list(extract_tag_from_result(result) or []) + except Exception: + tags_from_result = [] + + url_from_result = Add_File._get_url(result, pipe_obj) + + def _has_namespace_tag(tags: Sequence[str], namespace: str) -> bool: + namespace_text = str(namespace or "").strip().lower() + if not namespace_text: + return False + prefix = f"{namespace_text}:" + for tag in tags or []: + text = str(tag or "").strip().lower() + if text.startswith(prefix): + return True + return False + + def _extract_screenshot_time_title() -> tuple[Optional[str], Optional[str]]: + current_title = str(preferred_title or "").strip() + filename_title = str(media_path.stem or "").strip() + if current_title and current_title != filename_title: + return None, None + if not url_from_result: + return None, None + suffix = str(media_path.suffix or "").strip().lower() + if suffix not in {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif", ".mhtml"}: + return None, None + match = _SCREENSHOT_TIME_SUFFIX_RE.match(str(media_path.stem or "").strip()) + if not match: + return None, None + title_text = str(match.group("title") or "").strip().replace("_", " ").strip() + label_text = str(match.group("label") or "").strip().lower() + if not title_text or not label_text: + return None, None + return title_text, label_text + + preferred_title = pipe_obj.title + if not preferred_title: + for t in tags_from_result: + if str(t).strip().lower().startswith("title:"): + candidate = t.split(":", 1)[1].strip().replace("_", " ").strip() + if candidate: + preferred_title = candidate + break + if not preferred_title: + preferred_title = extract_title_from_result(result) + if preferred_title: + preferred_title = preferred_title.replace("_", " ").strip() + + derived_screenshot_title, derived_time_tag = _extract_screenshot_time_title() + if derived_screenshot_title and ( + not preferred_title or str(preferred_title or "").strip() == str(media_path.stem or "").strip() + ): + preferred_title = derived_screenshot_title + + store = getattr(pipe_obj, "store", None) + _, sidecar_hash, sidecar_tags, sidecar_url = Add_File._load_sidecar_bundle( + media_path, store, config + ) + + def normalize_title_tag(tag: str) -> str: + if str(tag).strip().lower().startswith("title:"): + parts = tag.split(":", 1) + if len(parts) == 2: + value = parts[1].replace("_", " ").strip() + return f"title:{value}" + return tag + + tags_from_result_no_title = [ + t for t in tags_from_result + if not str(t).strip().lower().startswith("title:") + ] + sidecar_tags = collapse_namespace_tags( + [normalize_title_tag(t) for t in sidecar_tags], + "title", + prefer="last" + ) + sidecar_tags_filtered = [ + t for t in sidecar_tags if not str(t).strip().lower().startswith("title:") + ] + + merged_tags = merge_sequences( + tags_from_result_no_title, + sidecar_tags_filtered, + case_sensitive=True + ) + + if derived_time_tag and not _has_namespace_tag(merged_tags, "time") and not _has_namespace_tag(merged_tags, "timestamp"): + merged_tags.append(f"time:{derived_time_tag}") + + if preferred_title: + merged_tags.append(f"title:{preferred_title}") + + merged_url = merge_sequences(url_from_result, sidecar_url, case_sensitive=False) + merged_url = [u for u in merged_url if Add_File._is_probable_url(u)] + + file_hash = Add_File._resolve_file_hash( + result, + media_path, + pipe_obj, + sidecar_hash + ) + + relationship_tags = [ + t for t in merged_tags + if isinstance(t, str) and t.strip().lower().startswith("relationship:") + ] + if relationship_tags: + try: + if (not isinstance(getattr(pipe_obj, "relationships", None), dict) or not pipe_obj.relationships): + king: Optional[str] = None + alts: List[str] = [] + for rel_tag in relationship_tags: + k, a = _parse_relationship_tag_king_alts(rel_tag) + if k and not king: + king = k + if a: + alts.extend(a) + if king: + seen_alt: set[str] = set() + alts = [ + h for h in alts if h and h != king and len(h) == 64 + and not (h in seen_alt or seen_alt.add(h)) + ] + payload: Dict[str, Any] = {"king": [king]} + if alts: + payload["alt"] = alts + pipe_obj.relationships = payload + except Exception: + pass + + merged_tags = [ + t for t in merged_tags if + not (isinstance(t, str) and t.strip().lower().startswith("relationship:")) + ] + + pipe_obj.tag = merged_tags + if preferred_title and not pipe_obj.title: + pipe_obj.title = preferred_title + if file_hash and not pipe_obj.hash: + pipe_obj.hash = file_hash + if isinstance(pipe_obj.extra, dict): + pipe_obj.extra["url"] = merged_url + return merged_tags, merged_url, preferred_title, file_hash diff --git a/cmdlet/file/add_validation.py b/cmdlet/file/add_validation.py new file mode 100644 index 0000000..fa4118f --- /dev/null +++ b/cmdlet/file/add_validation.py @@ -0,0 +1,611 @@ +"""File validation, path resolution, and source fetching for add-file.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, List +from pathlib import Path +import sys +import tempfile +import shutil +from urllib.parse import urlparse + +from SYS import models +from SYS import pipeline as ctx +from SYS.logger import log, debug +from SYS.pipeline_progress import PipelineProgress +from SYS.utils import sha256_file, sanitize_filename +from API.HTTP import download_direct_file + +from .. import _shared as sh + +# Import from add_core (safe: add_core defines these before importing this module) +from .add_core import ( + Add_File, + _CommandDependencies, + SUPPORTED_MEDIA_EXTENSIONS, + _REMOTE_URL_PREFIXES, +) + +coerce_to_path = sh.coerce_to_path +get_field = sh.get_field + + +def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]: + if not instance or not backend_name: + return None + try: + return instance[backend_name] + except Exception: + pass + target = str(backend_name or "").strip().lower() + if not target: + return None + try: + for candidate in instance.list_backends(): + if isinstance(candidate, str) and candidate.strip().lower() == target: + try: + return instance[candidate] + except Exception: + continue + except Exception: + pass + return None + + +def _build_provider_filename( + pipe_obj: models.PipeObject, + fallback_hash: Optional[str] = None, + source_url: Optional[str] = None, +) -> str: + title_candidates: List[str] = [] + title_value = getattr(pipe_obj, "title", "") + if title_value: + title_candidates.append(str(title_value)) + + extra = getattr(pipe_obj, "extra", {}) + if isinstance(extra, dict): + candid = extra.get("name") or extra.get("title") + if candid: + title_candidates.append(str(candid)) + + metadata = getattr(pipe_obj, "metadata", {}) + if isinstance(metadata, dict): + meta_name = metadata.get("title") or metadata.get("name") + if meta_name: + title_candidates.append(str(meta_name)) + + text = "" + for candidate in title_candidates: + if candidate: + text = candidate.strip() + if text: + break + + if not text and fallback_hash: + text = fallback_hash[:8] + + safe_name = sanitize_filename(text or "download") + + ext = "" + if isinstance(metadata, dict): + ext = metadata.get("ext") or metadata.get("extension") or "" + if not ext and isinstance(extra, dict): + ext = extra.get("ext") or "" + if not ext and source_url: + try: + parsed = urlparse(source_url) + ext = Path(parsed.path).suffix.lstrip(".") + except Exception: + ext = "" + + if ext: + ext_text = str(ext) + if not ext_text.startswith("."): + ext_text = "." + ext_text.lstrip(".") + if not safe_name.lower().endswith(ext_text.lower()): + safe_name = f"{safe_name}{ext_text}" + + return safe_name or "download" + + +def _maybe_download_backend_file( + backend: Any, + file_hash: str, + pipe_obj: models.PipeObject, + *, + output_dir: Optional[Path] = None, +) -> tuple[Optional[Path], Optional[Path]]: + """Best-effort fetch of a backend file when get_file returns a URL. + + Returns (downloaded_path, temp_dir_to_cleanup). + """ + + downloader = getattr(backend, "download_to_temp", None) + if not callable(downloader): + return None, None + + tmp_dir: Optional[Path] = None + try: + suffix = None + if pipe_obj.path: + try: + suffix = Path(pipe_obj.path).suffix + except Exception: + pass + + if not suffix: + metadata = getattr(pipe_obj, "metadata", {}) + if isinstance(metadata, dict): + suffix = metadata.get("ext") + + download_root = output_dir + if download_root is None: + tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) + download_root = tmp_dir + if download_root is None: + return None, None + + import inspect + + sig = inspect.signature(downloader) + kwargs = {"temp_root": download_root} + if "suffix" in sig.parameters: + kwargs["suffix"] = suffix + + pipeline_progress = PipelineProgress(ctx) + transfer_label = "peer transfer" + try: + transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label + except Exception: + transfer_label = "peer transfer" + if "pipeline_progress" in sig.parameters: + kwargs["pipeline_progress"] = pipeline_progress + if "transfer_label" in sig.parameters: + kwargs["transfer_label"] = transfer_label + if "progress_callback" in sig.parameters: + + def _cb(done, total): + try: + total_val = int(total) if total is not None else None + except Exception: + total_val = None + try: + if int(done or 0) <= 0: + pipeline_progress.begin_transfer( + label=transfer_label, + total=total_val, + ) + except Exception: + pass + try: + pipeline_progress.update_transfer( + label=transfer_label, + completed=int(done or 0), + total=total_val, + ) + except Exception: + pass + + kwargs["progress_callback"] = _cb + + downloaded = downloader(str(file_hash), **kwargs) + + if isinstance(downloaded, Path) and downloaded.exists(): + if output_dir is not None: + pipe_obj.is_temp = False + if isinstance(pipe_obj.extra, dict): + pipe_obj.extra["_direct_export_download"] = True + else: + pipe_obj.extra = {"_direct_export_download": True} + return downloaded, None + pipe_obj.is_temp = True + return downloaded, tmp_dir + except Exception: + pass + + if tmp_dir is not None: + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None + + +def _download_remote_backend_url( + remote_url: str, + pipe_obj: models.PipeObject, + *, + file_hash: Optional[str] = None, + output_dir: Optional[Path] = None, +) -> tuple[Optional[Path], Optional[Path]]: + """Best-effort fetch of a remote backend URL. + + Returns (downloaded_path, temp_dir_to_cleanup). + When ``output_dir`` is provided, the file is downloaded directly there and no + temp cleanup path is returned. + """ + + url_text = str(remote_url or "").strip() + if not url_text: + return None, None + if not url_text.lower().startswith(_REMOTE_URL_PREFIXES): + return None, None + if not url_text.lower().startswith(("http://", "https://")): + return None, None + + tmp_dir: Optional[Path] = None + try: + download_root = output_dir + if download_root is None: + tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) + download_root = tmp_dir + + suggested_name = _build_provider_filename( + pipe_obj, + fallback_hash=file_hash, + source_url=url_text, + ) + pipeline_progress = PipelineProgress(ctx) + try: + destination_label = str(download_root) if download_root is not None else "temporary workspace" + pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}") + except Exception: + pass + + downloaded = download_direct_file( + url_text, + download_root, + quiet=False, + suggested_filename=suggested_name, + pipeline_progress=pipeline_progress, + ) + downloaded_path = getattr(downloaded, "path", None) + if isinstance(downloaded_path, Path) and downloaded_path.exists(): + if output_dir is not None: + pipe_obj.is_temp = False + if isinstance(pipe_obj.extra, dict): + pipe_obj.extra["_direct_export_download"] = True + else: + pipe_obj.extra = {"_direct_export_download": True} + return downloaded_path, None + + pipe_obj.is_temp = True + return downloaded_path, tmp_dir + except Exception: + pass + finally: + try: + PipelineProgress(ctx).clear_status() + except Exception: + pass + + if tmp_dir is not None: + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None + + +def _maybe_download_plugin_result( + result: Any, + pipe_obj: models.PipeObject, + config: Dict[str, Any], + deps: Optional[_CommandDependencies] = None, +) -> tuple[Optional[Path], Optional[str], Optional[Path]]: + plugin_key = None + for source in ( + pipe_obj.plugin, + get_field(result, "plugin"), + get_field(result, "table"), + ): + candidate = Add_File._normalize_provider_key(source) + if candidate: + plugin_key = candidate + break + + if not plugin_key: + return None, None, None + + if deps is None: + deps = _CommandDependencies(config) + + plugin = deps.get_plugin(plugin_key) + if plugin is None: + return None, None, None + + try: + return plugin.resolve_pipe_result_download(result, pipe_obj) + except Exception as exc: + debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}") + + return None, None, None + + +def _download_piped_source( + pipe_obj: models.PipeObject, + config: Dict[str, Any], + store_instance: Optional[Any], + deps: Optional[_CommandDependencies] = None, +) -> tuple[Optional[Path], Optional[str], Optional[Path]]: + r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip() + r_store = str(getattr(pipe_obj, "store", None) or "").strip() + if not (r_hash and r_store): + return None, None, None + + if deps is None: + deps = _CommandDependencies(config) + + backend_registry = store_instance or deps.get_backend_registry() + backend = _resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None + if backend is None: + return None, None, None + + try: + source = backend.get_file(r_hash.lower()) + if isinstance(source, Path) and source.exists(): + pipe_obj.path = str(source) + return source, str(r_hash), None + if isinstance(source, str) and source.strip(): + dl_path, tmp_dir = _maybe_download_backend_file( + backend, str(r_hash), pipe_obj + ) + if dl_path and dl_path.exists(): + return dl_path, str(r_hash), tmp_dir + source_url = str(source).strip() + if source_url.lower().startswith(("http://", "https://")): + download_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) + try: + filename = _build_provider_filename( + pipe_obj, + str(r_hash), + source_url, + ) + downloaded = download_direct_file( + source_url, + download_dir, + quiet=True, + suggested_filename=filename, + ) + downloaded_path = downloaded.path + if downloaded_path and downloaded_path.exists(): + pipe_obj.is_temp = True + pipe_obj.path = str(downloaded_path) + return downloaded_path, str(r_hash), download_dir + except Exception as exc: + debug(f"[add-file] Provider download failed: {exc}") + try: + shutil.rmtree(download_dir, ignore_errors=True) + except Exception: + pass + except Exception: + pass + + return None, None, None + + +def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]: + """Scan a directory for supported media files and return list of file info dicts. + + Each dict contains: + - path: Path object + - name: filename + - hash: sha256 hash (or None if compute_hash=False) + - size: file size in bytes + - ext: file extension + """ + if not directory.exists() or not directory.is_dir(): + return [] + + files_info: List[Dict[str, Any]] = [] + + try: + for item in directory.iterdir(): + if not item.is_file(): + continue + + ext = item.suffix.lower() + if ext not in SUPPORTED_MEDIA_EXTENSIONS: + continue + + file_hash = None + if compute_hash: + try: + file_hash = sha256_file(item) + except Exception as exc: + debug(f"Failed to hash {item}: {exc}") + continue + + try: + size = item.stat().st_size + except Exception: + size = 0 + + files_info.append( + { + "path": item, + "name": item.name, + "hash": file_hash, + "size": size, + "ext": ext, + } + ) + except Exception as exc: + debug(f"Error scanning directory {directory}: {exc}") + + return files_info + + +def _validate_source(media_path: Optional[Path], allow_all_extensions: bool = False) -> bool: + """Validate that the source file exists and is supported. + + Args: + media_path: Path to the file to validate + allow_all_extensions: If True, skip file type filtering for non-backend exports. + If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest. + """ + if media_path is None: + return False + + if not media_path.exists() or not media_path.is_file(): + log(f"File not found: {media_path}") + return False + + if not allow_all_extensions: + file_extension = media_path.suffix.lower() + if file_extension not in SUPPORTED_MEDIA_EXTENSIONS: + log(f"❌ Unsupported file type: {file_extension}", file=sys.stderr) + return False + + return True + + +def _is_probable_url(s: Any) -> bool: + """Check if a string looks like a URL/magnet/identifier (vs a tag/title).""" + if not isinstance(s, str): + return False + val = s.strip().lower() + if not val: + return False + if val.startswith(_REMOTE_URL_PREFIXES): + return True + if "://" in val: + return True + if val.startswith("hash:"): + return False + return False + + +def _resolve_source( + result: Any, + source_arg: Optional[str], + pipe_obj: models.PipeObject, + config: Dict[str, Any], + export_destination: Optional[Path] = None, + store_instance: Optional[Any] = None, + deps: Optional[_CommandDependencies] = None, +) -> tuple[Optional[Path], Optional[str], Optional[Path]]: + """Resolve the source file path from the positional source arg or pipeline result. + + Returns (media_path, file_hash, temp_dir_to_cleanup). + """ + # PRIORITY 1a: Prefer an explicit local path when it exists. + if isinstance(result, dict): + r_path = result.get("path") + r_hash = result.get("hash") + if r_path: + try: + p = coerce_to_path(r_path) + if p.exists() and p.is_file(): + pipe_obj.path = str(p) + return p, str(r_hash) if r_hash else None, None + except Exception: + pass + + # PRIORITY 1b: Try hash+store from result (fetch from backend) + r_hash = get_field(result, "hash") or get_field(result, "file_hash") + r_store = get_field(result, "store") + + if r_hash and r_store: + try: + if deps is None: + deps = _CommandDependencies(config) + backend_registry = store_instance or deps.get_backend_registry() + + backend = _resolve_backend_by_name(backend_registry, r_store) + if backend is not None: + mp = backend.get_file(r_hash) + if isinstance(mp, Path) and mp.exists(): + pipe_obj.path = str(mp) + return mp, str(r_hash), None + if isinstance(mp, str) and mp.strip(): + try: + mp_path = Path(str(mp)) + except Exception: + mp_path = None + if mp_path is not None and mp_path.exists() and mp_path.is_file(): + pipe_obj.path = str(mp_path) + return mp_path, str(r_hash), None + + dl_path, tmp_dir = _maybe_download_backend_file( + backend, + str(r_hash), + pipe_obj, + output_dir=export_destination, + ) + if dl_path and dl_path.exists(): + pipe_obj.path = str(dl_path) + return dl_path, str(r_hash), tmp_dir + + dl_path, tmp_dir = _download_remote_backend_url( + str(mp), + pipe_obj, + file_hash=str(r_hash), + output_dir=export_destination, + ) + if dl_path and dl_path.exists(): + pipe_obj.path = str(dl_path) + return dl_path, str(r_hash), tmp_dir + except Exception as exc: + debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}") + + # PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result) + candidate: Optional[Path] = None + + if source_arg: + candidate = Path(source_arg) + elif pipe_obj.path: + candidate = Path(pipe_obj.path) + + if not candidate: + obj = result[0] if isinstance(result, list) and result else result + if obj: + try: + candidate = coerce_to_path(obj) + except ValueError: + pass + + if candidate: + s = str(candidate).lower() + if s.startswith(_REMOTE_URL_PREFIXES): + downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result( + result, + pipe_obj, + config, + deps=deps, + ) + if downloaded_path: + pipe_obj.path = str(downloaded_path) + return downloaded_path, hash_hint, tmp_dir + + dl_path, tmp_dir = _download_remote_backend_url( + str(candidate), + pipe_obj, + file_hash=get_field(result, "hash") or get_field(result, "file_hash"), + output_dir=export_destination, + ) + if dl_path: + pipe_obj.path = str(dl_path) + hash_hint = get_field(result, "hash") or get_field(result, "file_hash") + return dl_path, hash_hint, tmp_dir + + log("add-file could not auto-fetch remote source. Use download-file first.", file=sys.stderr) + return None, None, None + + pipe_obj.path = str(candidate) + hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None) + return candidate, hash_hint, None + + downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result( + result, + pipe_obj, + config, + deps=deps, + ) + if downloaded_path: + pipe_obj.path = str(downloaded_path) + return downloaded_path, hash_hint, tmp_dir + + debug(f"No resolution path matched. result type={type(result).__name__}") + log("File path could not be resolved") + return None, None, None diff --git a/cmdlet/file/download.py b/cmdlet/file/download.py index bd38068..69a128f 100644 --- a/cmdlet/file/download.py +++ b/cmdlet/file/download.py @@ -1,3101 +1,56 @@ -"""Generic file/stream downloader. +"""download-file cmdlet — re-export module. -Supports: -- Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp) -- Piped plugin items (uses plugin.download when available) -- Streaming sites via yt-dlp (YouTube, Bandcamp, etc.) +This module re-exports the public API from the split sub-modules to maintain +backward compatibility with existing imports. """ -from __future__ import annotations +from .download_core import Download_File, CMDLET -from collections.abc import Mapping, Sequence as SequenceABC -import sys -import re -from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence -from urllib.parse import urlparse -from contextlib import AbstractContextManager, nullcontext -import shutil -import webbrowser - - -from API.HTTP import download_direct_file -from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult -from SYS.logger import log, debug_panel, is_debug_enabled -from SYS.payload_builders import build_file_result_payload, build_table_result_payload -from SYS.pipeline_progress import PipelineProgress -from SYS.result_table import Table, build_display_row -from SYS.rich_display import stderr_console as get_stderr_console -from SYS import pipeline as pipeline_context -from SYS.item_accessors import get_result_title -from rich.prompt import Prompt -# SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid -# pulling in Cryptodome (~900ms) at module import time. -from SYS.selection_builder import ( - build_hash_store_selection, - extract_selection_fields, - extract_urls_from_selection_args, - selection_args_have_url, +# Re-export sub-module symbols for direct import if needed +from .download_fetch import ( + _emit_plugin_items, + _consume_plugin_download_result, + _process_explicit_urls, + _expand_provider_items, + _process_provider_items, + _download_provider_items, + _emit_local_file, + _resolve_provider_preflight_items, + _build_provider_playlist_item_selector, + _format_timecode, + _rebase_subtitle_timestamp_text, + _format_clip_range, + _apply_clip_decorations, + _parse_clip_spec_to_ranges, ) -from SYS.utils import sha256_file -try: - from plugins.ytdlp import YtDlpTool # type: ignore -except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers - YtDlpTool = None # type: ignore - -from .. import _shared as sh - -Cmdlet = sh.Cmdlet -CmdletArg = sh.CmdletArg -SharedArgs = sh.SharedArgs -QueryArg = sh.QueryArg -parse_cmdlet_args = sh.parse_cmdlet_args -register_url_with_local_library = sh.register_url_with_local_library -coerce_to_pipe_object = sh.coerce_to_pipe_object -get_field = sh.get_field -resolve_target_dir = sh.resolve_target_dir -coerce_to_path = sh.coerce_to_path -build_pipeline_preview = sh.build_pipeline_preview - -class Download_File(Cmdlet): - """Class-based download-file cmdlet - direct HTTP downloads.""" - - def __init__(self) -> None: - """Initialize download-file cmdlet.""" - super().__init__( - name="download-file", - summary="Download files or streaming media", - usage= - "download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]", - alias=["dl-file", - "download-http"], - arg=[ - SharedArgs.URL, - SharedArgs.PLUGIN, - SharedArgs.INSTANCE, - SharedArgs.PATH, - SharedArgs.QUERY, - CmdletArg( - name="name", - type="string", - description="Output filename override for store exports.", - ), - CmdletArg( - name="browser", - type="flag", - description="Open a backend-provided browser URL instead of exporting to disk when available.", - ), - QueryArg( - "clip", - key="clip", - aliases=["range", - "section", - "sections"], - type="string", - required=False, - description=( - "Clip time ranges via -query keyed fields (e.g. clip:1m-2m or clip:00:01-00:10). " - "Comma-separated values supported." - ), - query_only=True, - ), - CmdletArg( - name="item", - type="string", - description="Item selection for playlists/formats", - ), - ], - detail=[ - "Download files directly via HTTP or streaming media via yt-dlp.", - "Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.", - "Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.", - "For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.", - ], - exec=self.run, - ) - self.register() - - def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: - """Main execution method.""" - try: - debug_panel( - "download-file", - [ - ("args", list(args)), - ("has_piped_input", bool(result)), - ], - border_style="cyan", - ) - except Exception: - pass - return self._run_impl(result, args, config) - - @staticmethod - def _path_from_download_result(result_obj: Any) -> Path: - """Normalize downloader return values to a concrete filesystem path.""" - resolved = coerce_to_path(result_obj) - if resolved is None: - raise DownloadError("Could not determine downloaded file path") - return resolved - - @staticmethod - def _selection_run_label( - run_args: Sequence[str], - *, - extra_url_prefixes: Sequence[str] = (), - ) -> str: - try: - urls = extract_urls_from_selection_args( - run_args, - extra_url_prefixes=extra_url_prefixes, - ) - if urls: - return str(urls[0]) - except Exception: - pass - - for arg in run_args: - text = str(arg or "").strip() - if text and not text.startswith("-"): - return text - return "item" - - @staticmethod - def _batch_progress_state(config: Optional[Dict[str, Any]]) -> tuple[bool, int, int, str]: - if not isinstance(config, dict): - return False, 0, 0, "" - - suppress_nested = bool(config.get("_download_file_suppress_nested_pipe_progress")) - if not suppress_nested: - return False, 0, 0, "" - - try: - total = max(0, int(config.get("_download_file_batch_total") or 0)) - except Exception: - total = 0 - try: - index = max(0, int(config.get("_download_file_batch_index") or 0)) - except Exception: - index = 0 - try: - label = str(config.get("_download_file_batch_label") or "").strip() - except Exception: - label = "" - - return True, total, index, label - - @staticmethod - def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]: - loader = registry.get("list_selection_url_prefixes") - if not callable(loader): - return [] - try: - values = loader() or [] - except Exception: - return [] - return [str(value).strip().lower() for value in values if str(value or "").strip()] - - def _emit_plugin_items( - self, - *, - items: Sequence[Any], - config: Dict[str, Any], - ) -> int: - emitted = 0 - for item in items: - if not isinstance(item, dict): - continue - pipeline_context.emit(item) - if item.get("url"): - try: - pipe_obj = coerce_to_pipe_object(item) - register_url_with_local_library(pipe_obj, config) - except Exception: - pass - emitted += 1 - return emitted - - def _consume_plugin_download_result( - self, - *, - result: Any, - config: Dict[str, Any], - ) -> tuple[int, Optional[int], bool]: - if result is None: - return 0, None, False - - if isinstance(result, list): - if result and all(isinstance(item, dict) for item in result): - return self._emit_plugin_items(items=result, config=config), 0, True - return 0, None, False - - if not isinstance(result, dict): - return 0, None, False - - action = str( - result.get("action") - or result.get("plugin_action") - or "" - ).strip().lower() - - if action in {"emit_items", "emit_pipe_objects"}: - items = result.get("items") or [] - exit_code = result.get("exit_code") - emitted = self._emit_plugin_items( - items=items if isinstance(items, list) else [], - config=config, - ) - try: - normalized_exit = int(exit_code) if exit_code is not None else 0 - except Exception: - normalized_exit = 0 - return emitted, normalized_exit, True - - if action == "handled": - exit_code = result.get("exit_code") - try: - normalized_exit = int(exit_code) if exit_code is not None else 0 - except Exception: - normalized_exit = 0 - try: - downloaded = int(result.get("downloaded") or 0) - except Exception: - downloaded = 0 - return downloaded, normalized_exit, True - - return 0, None, False - - def _process_explicit_urls( - self, - *, - raw_urls: Sequence[str], - final_output_dir: Path, - config: Dict[str, - Any], - quiet_mode: bool, - registry: Dict[str, - Any], - progress: PipelineProgress, - parsed: Dict[str, Any], - args: Sequence[str], - context_items: Sequence[Any] = (), - ) -> tuple[int, - Optional[int]]: - downloaded_count = 0 - skipped_duplicate_only = 0 - attempted_download = False - suppress_nested, batch_total, batch_index, batch_label = self._batch_progress_state(config) - total_urls = len(raw_urls or []) - - try: - if total_urls > 1 and not suppress_nested: - progress.begin_pipe(total_items=total_urls, items_preview=list(raw_urls[:5])) - except Exception: - pass - - SearchResult = registry.get("SearchResult") - get_plugin = registry.get("get_plugin") - match_plugin_name_for_url = registry.get("match_plugin_name_for_url") - - for idx, url in enumerate(raw_urls, 1): - try: - try: - display_total = batch_total if batch_total > 0 else total_urls - display_index = batch_index if batch_total > 0 else idx - display_label = batch_label or str(url) - if display_total > 0: - progress.set_status( - f"downloading {display_index}/{display_total}: {display_label}" - ) - except Exception: - pass - - # Check providers first - provider_name = None - if match_plugin_name_for_url: - try: - provider_name = match_plugin_name_for_url(str(url)) - except Exception: - pass - - provider = None - if provider_name and get_plugin: - provider = get_plugin(provider_name, config) - - if provider: - try: - # Try generic handle_url - handled = False - if hasattr(provider, "handle_url"): - try: - handled, path = provider.handle_url(str(url), output_dir=final_output_dir) - if handled: - extra_meta = None - title_hint = None - tags_hint: Optional[List[str]] = None - media_kind_hint = None - path_value: Optional[Any] = path - - if isinstance(path, dict): - plugin_action = str( - path.get("action") - or path.get("plugin_action") - or "" - ).strip().lower() - if plugin_action == "download_items" or bool(path.get("download_items")): - request_metadata = path.get("metadata") or path.get("full_metadata") or {} - if not isinstance(request_metadata, dict): - request_metadata = {} - magnet_id = path.get("magnet_id") or request_metadata.get("magnet_id") - if magnet_id is not None: - request_metadata.setdefault("magnet_id", magnet_id) - - if SearchResult is None: - continue - - sr = SearchResult( - table=str(provider_name), - title=str(path.get("title") or path.get("name") or f"{provider_name} item"), - path=str(path.get("path") or path.get("url") or url), - full_metadata=request_metadata, - ) - downloaded_extra = self._download_provider_items( - provider=provider, - provider_name=str(provider_name), - search_result=sr, - output_dir=final_output_dir, - progress=progress, - quiet_mode=quiet_mode, - config=config, - ) - if downloaded_extra: - downloaded_count += int(downloaded_extra) - continue - - plugin_downloaded, plugin_exit, plugin_handled = self._consume_plugin_download_result( - result=path, - config=config, - ) - if plugin_handled: - downloaded_count += plugin_downloaded - if plugin_exit is not None and plugin_downloaded == 0: - return downloaded_count, int(plugin_exit) - if plugin_downloaded: - continue - - path_value = path.get("path") or path.get("file_path") - extra_meta = path.get("metadata") or path.get("full_metadata") - title_hint = path.get("title") or path.get("name") - media_kind_hint = path.get("media_kind") - tags_val = path.get("tags") or path.get("tag") - if isinstance(tags_val, (list, tuple, set)): - tags_hint = [str(t) for t in tags_val if t] - elif isinstance(tags_val, str) and tags_val.strip(): - tags_hint = [str(tags_val).strip()] - - if path_value: - p_val = Path(str(path_value)) - if not title_hint and isinstance(extra_meta, dict): - title_hint = extra_meta.get("title") or extra_meta.get("name") - - self._emit_local_file( - downloaded_path=p_val, - source=str(url), - title_hint=str(title_hint) if title_hint else p_val.stem, - tags_hint=tags_hint, - media_kind_hint=str(media_kind_hint) if media_kind_hint else "file", - full_metadata=extra_meta, - progress=progress, - config=config, - provider_hint=provider_name - ) - downloaded_count += 1 - continue - except Exception as e: - debug_panel( - "download-file provider error", - [ - ("plugin", provider_name), - ("url", url), - ("operation", "handle_url"), - ("error", e), - ], - border_style="yellow", - ) - - # Try generic download_url if not already handled - if not handled and hasattr(provider, "download_url"): - parsed_for_provider = parsed - provider_preflight_items = self._resolve_provider_preflight_items( - provider, - url=str(url), - parsed=parsed, - args=args, - ) - if provider_preflight_items: - provider_preflight_urls = [ - str(item.get("url") or "").strip() - for item in provider_preflight_items - if str(item.get("url") or "").strip() - ] - provider_preflight_urls, preflight_exit, provider_skipped = self._preflight_explicit_url_duplicates( - raw_urls=provider_preflight_urls, - config=config, - ) - if preflight_exit is not None: - return downloaded_count, int(preflight_exit) - if provider_skipped: - if not provider_preflight_urls: - skipped_duplicate_only += 1 - continue - selector = self._build_provider_playlist_item_selector( - provider_preflight_items, - remaining_urls=provider_preflight_urls, - ) - if selector: - parsed_for_provider = dict(parsed) - parsed_for_provider["item"] = selector - try: - attempted_download = True - res = provider.download_url( - str(url), - final_output_dir, - parsed=parsed_for_provider, - args=list(args), - progress=progress, - quiet_mode=quiet_mode, - context_items=list(context_items or []), - ) - except TypeError: - attempted_download = True - res = provider.download_url(str(url), final_output_dir) - - plugin_downloaded, plugin_exit, plugin_handled = self._consume_plugin_download_result( - result=res, - config=config, - ) - if plugin_handled: - downloaded_count += plugin_downloaded - if plugin_exit is not None and plugin_downloaded == 0: - return downloaded_count, int(plugin_exit) - if plugin_downloaded: - continue - - if res: - # Standardize result: can be Path, tuple(Path, Info), or dict with "path" - p_val = None - extra_meta = None - if isinstance(res, (str, Path)): - p_val = Path(res) - elif isinstance(res, tuple) and len(res) > 0: - p_val = Path(res[0]) - if len(res) > 1 and isinstance(res[1], dict): - extra_meta = res[1] - elif isinstance(res, dict): - path_candidate = res.get("path") or res.get("file_path") - if path_candidate: - p_val = Path(path_candidate) - extra_meta = res - - if p_val: - self._emit_local_file( - downloaded_path=p_val, - source=str(url), - title_hint=p_val.stem, - tags_hint=None, - media_kind_hint=extra_meta.get("media_kind") if extra_meta else "file", - full_metadata=extra_meta, - provider_hint=provider_name, - progress=progress, - config=config, - ) - downloaded_count += 1 - continue - - except Exception as e: - log(f"Provider {provider_name} error handling {url}: {e}", file=sys.stderr) - pass - - if not handled: - continue - - # Direct Download Fallback - attempted_download = True - result_obj = download_direct_file( - str(url), - final_output_dir, - quiet=quiet_mode, - pipeline_progress=progress, - ) - downloaded_path = self._path_from_download_result(result_obj) - - self._emit_local_file( - downloaded_path=downloaded_path, - source=str(url), - title_hint=downloaded_path.stem, - tags_hint=[f"title:{downloaded_path.stem}"], - media_kind_hint="file", - full_metadata=None, - progress=progress, - config=config, - ) - downloaded_count += 1 - - except DownloadError as e: - log(f"Download failed for {url}: {e}", file=sys.stderr) - except Exception as e: - log(f"Error processing {url}: {e}", file=sys.stderr) - - if downloaded_count == 0 and skipped_duplicate_only > 0 and not attempted_download: - return downloaded_count, 0 - return downloaded_count, None - - def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]: - if value is None: - return None - try: - normalized = str(value).strip() - except Exception: - return None - if not normalized: - return None - if "." in normalized: - normalized = normalized.split(".", 1)[0] - return normalized.lower() - - def _provider_key_from_item(self, item: Any) -> Optional[str]: - table_hint = get_field(item, "table") - key = self._normalize_provider_key(table_hint) - if key: - return key - provider_hint = get_field(item, "plugin") - key = self._normalize_provider_key(provider_hint) - if key: - return key - return self._normalize_provider_key(get_field(item, "source")) - - def _expand_provider_items( - self, - *, - piped_items: Sequence[Any], - registry: Dict[str, - Any], - config: Dict[str, - Any], - ) -> List[Any]: - get_provider = registry.get("get_plugin") - expanded_items: List[Any] = [] - - for item in piped_items: - try: - provider_key = self._provider_key_from_item(item) - provider = get_provider(provider_key, config) if provider_key and get_provider else None - - # Generic hook: If provider has expand_item(item), use it. - if provider and hasattr(provider, "expand_item") and callable(provider.expand_item): - try: - sub_items = provider.expand_item(item) - if sub_items: - expanded_items.extend(sub_items) - continue - except Exception as e: - debug_panel( - "download-file expand_item failed", - [ - ("plugin", provider_key), - ("error", e), - ], - border_style="yellow", - ) - - expanded_items.append(item) - except Exception: - expanded_items.append(item) - - return expanded_items - - def _process_provider_items(self, - *, - piped_items: Sequence[Any], - final_output_dir: Path, - config: Dict[str, - Any], - quiet_mode: bool, - registry: Dict[str, - Any], - progress: PipelineProgress, - ) -> tuple[int, int]: - downloaded_count = 0 - queued_magnet_submissions = 0 - get_provider = registry.get("get_plugin") - SearchResult = registry.get("SearchResult") - - expanded_items = self._expand_provider_items( - piped_items=piped_items, - registry=registry, - config=config - ) - - total_items = len(expanded_items) - processed_items = 0 - - try: - if total_items: - progress.set_percent(0) - except Exception: - pass - - for idx, item in enumerate(expanded_items, 1): - try: - label = "item" - table = get_field(item, "table") - title = get_field(item, "title") - target = get_field(item, "path") or get_field(item, "url") - - media_kind = get_field(item, "media_kind") - tags_val = get_field(item, "tag") - tags_list: Optional[List[str]] - if isinstance(tags_val, (list, set)): - tags_list = sorted([str(t) for t in tags_val if t]) - else: - tags_list = None - - full_metadata = get_field(item, "full_metadata") - if ((not full_metadata) and isinstance(item, - dict) - and isinstance(item.get("extra"), - dict)): - extra_md = item["extra"].get("full_metadata") - if isinstance(extra_md, dict): - full_metadata = extra_md - - try: - label = title or target - label = str(label or "item").strip() - if total_items: - pct = int(round((processed_items / max(1, total_items)) * 100)) - progress.set_percent(pct) - progress.set_status( - f"downloading {processed_items + 1}/{total_items}: {label}" - ) - except Exception: - pass - - transfer_label = label - - # If this looks like a plugin-owned item and a plugin is available, prefer plugin.download(). - downloaded_path: Optional[Path] = None - attempted_provider_download = False - provider_sr = None - provider_obj = None - provider_key = self._provider_key_from_item(item) - if provider_key and get_provider and SearchResult: - # Reuse helper to derive the plugin key from table/plugin/source hints. - provider_obj = get_provider(provider_key, config) - - if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): - try: - progress.begin_transfer(label=transfer_label, total=None) - except Exception: - pass - - if provider_obj is not None: - attempted_provider_download = True - sr = SearchResult( - table=str(table), - title=str(title or "Unknown"), - path=str(target or ""), - tag=set(tags_list) if tags_list else set(), - media_kind=str(media_kind or "file"), - full_metadata=full_metadata - if isinstance(full_metadata, dict) else {}, - ) - - # Preserve plugin-managed output structure when a plugin encodes nested paths. - output_dir = final_output_dir - # Generic: allow provider to strict output_dir? - # Using default output_dir for now. - - downloaded_path = provider_obj.download(sr, output_dir) - provider_sr = sr - - if downloaded_path is None: - try: - downloaded_extra = self._download_provider_items( - provider=provider_obj, - provider_name=str(provider_key), - search_result=sr, - output_dir=output_dir, - progress=progress, - quiet_mode=quiet_mode, - config=config, - ) - except Exception: - downloaded_extra = 0 - - if downloaded_extra: - downloaded_count += int(downloaded_extra) - continue - - # Fallback: if we have a direct HTTP URL and no provider successfully handled it - if (downloaded_path is None and not attempted_provider_download - and isinstance(target, str) and target.startswith("http")): - - suggested_name = str(title).strip() if title is not None else None - result_obj = download_direct_file( - target, - final_output_dir, - quiet=quiet_mode, - suggested_filename=suggested_name, - pipeline_progress=progress, - ) - downloaded_path = coerce_to_path(result_obj) - - if downloaded_path is None: - log( - f"Cannot download item (no provider handler / unsupported target): {title or target}", - file=sys.stderr, - ) - continue - - # Allow plugins to add or enrich tags and metadata during download. - if provider_sr is not None: - try: - sr_md = getattr(provider_sr, "full_metadata", None) - if isinstance(sr_md, dict) and sr_md: - full_metadata = sr_md - except Exception: - pass - - try: - if isinstance(full_metadata, dict): - t = str(full_metadata.get("title") or "").strip() - if t: - title = t - except Exception: - pass - - # Prefer tags from the search result object if the provider mutated them during download. - try: - sr_tags = getattr(provider_sr, "tag", None) - if isinstance(sr_tags, (set, list)) and sr_tags: - # Re-sync tags_list with the potentially enriched provider_sr.tag - tags_list = sorted([str(t) for t in sr_tags if t]) - except Exception: - pass - - self._emit_local_file( - downloaded_path=downloaded_path, - source=str(target) if target else None, - title_hint=str(title) if title else downloaded_path.stem, - tags_hint=tags_list, - media_kind_hint=str(media_kind) if media_kind else None, - full_metadata=full_metadata if isinstance(full_metadata, - dict) else None, - progress=progress, - config=config, - provider_hint=provider_key - ) - downloaded_count += 1 - - except DownloadError as e: - log(f"Download failed: {e}", file=sys.stderr) - except Exception as e: - log(f"Error downloading item: {e}", file=sys.stderr) - finally: - if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): - try: - progress.finish_transfer(label=transfer_label) - except Exception: - pass - processed_items += 1 - try: - pct = int(round((processed_items / max(1, total_items)) * 100)) - progress.set_percent(pct) - if processed_items >= total_items: - progress.clear_status() - except Exception: - pass - - return downloaded_count, queued_magnet_submissions - - def _download_provider_items( - self, - *, - provider: Any, - provider_name: str, - search_result: Any, - output_dir: Path, - progress: PipelineProgress, - quiet_mode: bool, - config: Dict[str, Any], - ) -> int: - if provider is None or not hasattr(provider, "download_items"): - return 0 - - def _on_emit(path: Path, file_url: str, relpath: str, metadata: Dict[str, Any]) -> None: - title_hint = None - try: - title_hint = metadata.get("name") or relpath - except Exception: - title_hint = relpath - title_hint = title_hint or (Path(path).name if path else "download") - - self._emit_local_file( - downloaded_path=path, - source=file_url, - title_hint=title_hint, - tags_hint=None, - media_kind_hint="file", - full_metadata=metadata if isinstance(metadata, dict) else None, - progress=progress, - config=config, - provider_hint=provider_name, - ) - - try: - downloaded_count = provider.download_items( - search_result, - output_dir, - emit=_on_emit, - progress=progress, - quiet_mode=quiet_mode, - path_from_result=coerce_to_path, - config=config, - ) - except TypeError: - downloaded_count = provider.download_items( - search_result, - output_dir, - emit=_on_emit, - progress=progress, - quiet_mode=quiet_mode, - path_from_result=coerce_to_path, - ) - except Exception as exc: - log(f"Provider {provider_name} download_items error: {exc}", file=sys.stderr) - return 0 - - try: - return int(downloaded_count or 0) - except Exception: - return 0 - - def _emit_local_file( - self, - *, - downloaded_path: Path, - source: Optional[str], - title_hint: Optional[str], - tags_hint: Optional[List[str]], - media_kind_hint: Optional[str], - full_metadata: Optional[Dict[str, Any]], - progress: PipelineProgress, - config: Dict[str, Any], - provider_hint: Optional[str] = None, - ) -> None: - title_val = (title_hint or downloaded_path.stem or "Unknown").strip() or downloaded_path.stem - hash_value = sha256_file(downloaded_path) - notes: Optional[Dict[str, str]] = None - try: - if isinstance(full_metadata, dict): - # Plugins attach pre-built notes under the generic "_notes" key - # (e.g. Tidal sets {"lyric": subtitles} during download enrichment). - # This keeps plugin-specific metadata handling inside the plugin. - _provider_notes = full_metadata.get("_notes") - if isinstance(_provider_notes, dict) and _provider_notes: - notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v} - except Exception: - notes = None - tag: List[str] = [] - if tags_hint: - tag.extend([str(t) for t in tags_hint if t]) - if not any(str(t).lower().startswith("title:") for t in tag): - tag.insert(0, f"title:{title_val}") - - payload: Dict[str, Any] = { - "path": str(downloaded_path), - "hash": hash_value, - "title": title_val, - "action": "cmdlet:download-file", - "download_mode": "file", - "store": "local", - "media_kind": media_kind_hint or "file", - "tag": tag, - } - if provider_hint: - payload["plugin"] = str(provider_hint) - if full_metadata: - payload["metadata"] = full_metadata - if notes: - payload["notes"] = notes - if source and str(source).startswith("http"): - payload["url"] = source - elif source: - payload["source_url"] = source - - pipeline_context.emit(payload) - - @staticmethod - def _path_looks_local(value: Any) -> bool: - text = str(value or "").strip() - if not text: - return False - if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")): - return False - if len(text) >= 2 and text[1] == ":": - return True - if text.startswith(("\\", "/", ".", "~")): - return True - return Path(text).exists() - - @staticmethod - def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str: - candidates = [ - get_result_title(result, "title", "name", "filename"), - get_result_title(metadata or {}, "title", "name", "filename"), - ] - for candidate in candidates: - if candidate is None: - continue - text = str(candidate).strip() - if text: - return text - return "" - - @staticmethod - def _sanitize_export_filename(name: str) -> str: - allowed_chars: List[str] = [] - for ch in str(name or ""): - if ch.isalnum() or ch in {"-", "_", " ", "."}: - allowed_chars.append(ch) - else: - allowed_chars.append(" ") - sanitized = " ".join("".join(allowed_chars).split()) - return sanitized or "export" - - @staticmethod - def _unique_export_path(path: Path) -> Path: - if not path.exists(): - return path - stem = path.stem - suffix = path.suffix - parent = path.parent - counter = 1 - while True: - candidate = parent / f"{stem} ({counter}){suffix}" - if not candidate.exists(): - return candidate - counter += 1 - - @staticmethod - def _iter_storage_export_refs( - parsed: Dict[str, Any], - piped_items: Sequence[Any], - ) -> tuple[List[Dict[str, Any]], List[Any], Optional[int]]: - refs: List[Dict[str, Any]] = [] - residual_items: List[Any] = [] - - query_text = str(parsed.get("query") or "").strip() - query_hash: Optional[str] = None - if query_text: - query_hash = sh.parse_single_hash_query(query_text) - if query_text.lower().startswith("hash") and not query_hash: - log('Error: -query must be of the form hash:<sha256>', file=sys.stderr) - return [], list(piped_items or []), 1 - - explicit_store = str(parsed.get("instance") or "").strip() - if query_hash: - if not explicit_store: - log('Error: No store name provided', file=sys.stderr) - return [], list(piped_items or []), 1 - refs.append( - { - "hash": query_hash, - "store": explicit_store, - "result": None, - } - ) - - for item in piped_items or []: - normalized_hash = sh.normalize_hash( - str(get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "") - ) - store_name = str(parsed.get("instance") or get_field(item, "store") or "").strip() - if normalized_hash and store_name: - refs.append( - { - "hash": normalized_hash, - "store": store_name, - "result": item, - } - ) - else: - residual_items.append(item) - - return refs, residual_items, None - - def _export_store_file( - self, - *, - file_hash: str, - store_name: str, - result: Any, - parsed: Dict[str, Any], - config: Dict[str, Any], - final_output_dir: Path, - ) -> int: - output_path = parsed.get("path") - explicit_output_requested = bool(output_path) - output_name = parsed.get("name") - browser_flag = bool(parsed.get("browser")) - - backend, _store_registry, _exc = sh.get_preferred_store_backend( - config, - store_name, - suppress_debug=True, - ) - if backend is None: - log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr) - return 1 - - metadata = backend.get_metadata(file_hash) - if not metadata: - log(f"Error: File metadata not found for hash {file_hash}", file=sys.stderr) - return 1 - - try: - debug_panel( - "download-file store export", - [ - ("hash", file_hash), - ("instance", store_name), - ("output_path", output_path or "<default>"), - ("output_name", output_name or "<auto>"), - ("browser", browser_flag), - ], - border_style="blue", - ) - except Exception: - pass - - want_url = browser_flag - source_path = backend.get_file(file_hash, url=want_url) - download_url = None - if isinstance(source_path, str): - if source_path.startswith(("http://", "https://")): - download_url = source_path - else: - source_path = Path(source_path) - - if download_url and (browser_flag or not explicit_output_requested): - try: - webbrowser.open(download_url) - except Exception as exc: - log(f"Error opening browser: {exc}", file=sys.stderr) - return 1 - - pipeline_context.emit( - build_file_result_payload( - title=self._resolve_display_title(result, metadata) or "Opened", - hash_value=file_hash, - store=store_name, - url=download_url, - ) - ) - return 0 - - if download_url is None: - if not source_path or not Path(source_path).exists(): - log(f"Error: Backend could not retrieve file for hash {file_hash}", file=sys.stderr) - return 1 - - filename = str(output_name or "").strip() - if not filename: - title = (metadata.get("title") if isinstance(metadata, dict) else None) or self._resolve_display_title(result, metadata) or "export" - filename = self._sanitize_export_filename(str(title)) - - ext = metadata.get("ext") if isinstance(metadata, dict) else None - if ext and not filename.endswith(str(ext)): - ext_text = str(ext) - if not ext_text.startswith("."): - ext_text = "." + ext_text - filename += ext_text - - if download_url: - result_obj = download_direct_file( - download_url, - final_output_dir, - quiet=True, - suggested_filename=filename, - pipeline_progress=config.get("_pipeline_progress") if isinstance(config, dict) else None, - ) - dest_path = self._path_from_download_result(result_obj) - else: - dest_path = self._unique_export_path(final_output_dir / filename) - shutil.copy2(Path(source_path), dest_path) - - pipeline_context.emit( - build_file_result_payload( - title=filename, - hash_value=file_hash, - store=store_name, - path=str(dest_path), - ) - ) - return 0 - - def _process_storage_items( - self, - *, - piped_items: Sequence[Any], - parsed: Dict[str, Any], - config: Dict[str, Any], - final_output_dir: Path, - ) -> tuple[int, List[Any], Optional[int]]: - refs, residual_items, early_exit = self._iter_storage_export_refs(parsed, piped_items) - if early_exit is not None: - return 0, list(residual_items), early_exit - if not refs: - return 0, list(residual_items), None - - successes = 0 - for ref in refs: - exit_code = self._export_store_file( - file_hash=str(ref.get("hash") or ""), - store_name=str(ref.get("store") or ""), - result=ref.get("result"), - parsed=parsed, - config=config, - final_output_dir=final_output_dir, - ) - if exit_code != 0: - return successes, list(residual_items), exit_code - successes += 1 - - return successes, list(residual_items), None - - def _process_explicit_local_sources( - self, - *, - local_sources: Sequence[str], - final_output_dir: Path, - parsed: Dict[str, Any], - progress: PipelineProgress, - config: Dict[str, Any], - ) -> int: - explicit_output_requested = bool(parsed.get("path")) - downloaded_count = 0 - for raw_source in local_sources or []: - source_path = Path(str(raw_source or "")).expanduser() - if not source_path.exists() or not source_path.is_file(): - log(f"File not found: {source_path}", file=sys.stderr) - continue - - if explicit_output_requested: - destination = final_output_dir / source_path.name - destination = self._unique_export_path(destination) - shutil.copy2(source_path, destination) - emit_path = destination - else: - emit_path = source_path - - self._emit_local_file( - downloaded_path=emit_path, - source=str(source_path), - title_hint=emit_path.stem, - tags_hint=None, - media_kind_hint="file", - full_metadata=None, - progress=progress, - config=config, - ) - downloaded_count += 1 - return downloaded_count - - def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None: - try: - stage_ctx = pipeline_context.get_stage_context() - except Exception: - stage_ctx = None - - is_last_stage = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) - if not is_last_stage: - return - - try: - quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False - except Exception: - quiet_mode = False - if quiet_mode: - return - - emitted_items: List[Any] = [] - try: - emitted_items = list(getattr(stage_ctx, "emits", None) or []) if stage_ctx is not None else [] - except Exception: - emitted_items = [] - - if not emitted_items: - return - - # Stop the live pipeline progress UI before rendering the details panel. - try: - live_progress = pipeline_context.get_live_progress() - except Exception: - live_progress = None - - if live_progress is not None: - try: - pipe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None - if isinstance(pipe_idx, int): - live_progress.finish_pipe(int(pipe_idx), force_complete=True) - except Exception: - pass - try: - live_progress.stop() - except Exception: - pass - try: - if hasattr(pipeline_context, "set_live_progress"): - pipeline_context.set_live_progress(None) - except Exception: - pass - - try: - subject = emitted_items[0] if len(emitted_items) == 1 else list(emitted_items) - # Use helper to display items and make them @-selectable - from .._shared import display_and_persist_items - display_and_persist_items(list(emitted_items), title="Result", subject=subject) - except Exception: - pass - - # Prevent CLI from printing a redundant table after the detail panels. - try: - if stage_ctx is not None: - stage_ctx.emits = [] - except Exception: - pass - - @staticmethod - def _load_provider_registry() -> Dict[str, Any]: - """Lightweight accessor for plugin helpers without hard dependencies.""" - try: - from PluginCore import registry as provider_registry # type: ignore - from PluginCore.base import SearchResult # type: ignore - - return { - "get_plugin": getattr(provider_registry, "get_plugin", None), - "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None), - "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None), - "SearchResult": SearchResult, - } - except Exception: - return { - "get_plugin": None, - "match_plugin_name_for_url": None, - "list_selection_url_prefixes": None, - "SearchResult": None, - } - - @classmethod - def _extract_hash_from_search_hit(cls, hit: Any) -> Optional[str]: - if not isinstance(hit, dict): - return None - for key in ("hash", "hash_hex", "file_hash", "hydrus_hash"): - v = hit.get(key) - normalized = sh.normalize_hash(str(v) if v is not None else None) - if normalized: - return normalized - return None - - @staticmethod - def _iter_duplicate_tag_values(item: Any) -> List[str]: - def _append_tag(out: List[str], value: Any) -> None: - text = "" - if isinstance(value, bytes): - try: - text = value.decode("utf-8", errors="ignore") - except Exception: - text = str(value) - elif isinstance(value, str): - text = value - if not text: - return - cleaned = text.strip() - if cleaned: - out.append(cleaned) - - def _collect_current(container: Any, out: List[str]) -> None: - if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)): - for tag in container: - _append_tag(out, tag) - return - if not isinstance(container, Mapping): - return - current = container.get("0") - if current is None: - current = container.get(0) - if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)): - for tag in current: - _append_tag(out, tag) - - def _collect_service_data(service_data: Any, out: List[str]) -> None: - if not isinstance(service_data, Mapping): - return - for key in ( - "display_tags", - "display_friendly_tags", - "display", - "storage_tags", - "statuses_to_tags", - "tags", - ): - _collect_current(service_data.get(key), out) - - collected: List[str] = [] - for raw_tags in ( - get_field(item, "tags_flat"), - get_field(item, "tags"), - get_field(item, "tag"), - ): - if isinstance(raw_tags, str): - _append_tag(collected, raw_tags) - continue - if isinstance(raw_tags, (list, tuple, set)): - for raw_tag in raw_tags: - _append_tag(collected, raw_tag) - continue - if not isinstance(raw_tags, Mapping): - continue - - statuses_map = raw_tags.get("service_keys_to_statuses_to_tags") - if isinstance(statuses_map, Mapping): - for status_payload in statuses_map.values(): - _collect_current(status_payload, collected) - - names_map = raw_tags.get("service_keys_to_names") - if isinstance(names_map, Mapping): - _ = names_map - - _collect_service_data(raw_tags, collected) - for maybe_service in raw_tags.values(): - _collect_service_data(maybe_service, collected) - - deduped: List[str] = [] - seen: set[str] = set() - for raw_tag in collected: - text = str(raw_tag or "").strip() - key = text.lower() - if not text or key in seen: - continue - seen.add(key) - deduped.append(text) - return deduped - - @staticmethod - def _extract_duplicate_namespace_tags(item: Any) -> List[str]: - tag_values = Download_File._iter_duplicate_tag_values(item) - - namespace_tags: List[str] = [] - seen: set[str] = set() - for raw_tag in tag_values: - text = str(raw_tag or "").strip() - if not text: - continue - lower = text.lower() - if ":" not in text or lower.startswith("title:"): - continue - if lower in seen: - continue - seen.add(lower) - namespace_tags.append(text) - return namespace_tags - - @staticmethod - def _extract_duplicate_title_tag(item: Any) -> Optional[str]: - for raw_tag in Download_File._iter_duplicate_tag_values(item): - tag_text = str(raw_tag or "").strip() - if not tag_text or not tag_text.lower().startswith("title:"): - continue - value = tag_text.split(":", 1)[1].strip() - if value: - return value - return None - - @classmethod - def _extract_duplicate_title(cls, item: Any) -> str: - for key in ("title", "name", "filename", "target"): - value = get_field(item, key) - text = str(value or "").strip() - if text: - return text - - tag_title = cls._extract_duplicate_title_tag(item) - if tag_title: - return tag_title - - path_value = str(get_field(item, "path") or "").strip() - if path_value and not path_value.lower().startswith(("http://", "https://", "file://")): - return path_value - - return "(exists)" - - @classmethod - def _has_duplicate_title(cls, item: Any) -> bool: - return cls._extract_duplicate_title(item) != "(exists)" - - @staticmethod - def _normalize_duplicate_preflight_policy(value: Any) -> Optional[str]: - text = str(value or "").strip().lower() - if not text: - return "skip" - mapping = { - "i": "ignore", - "ignore": "ignore", - "s": "skip", - "skip": "skip", - "c": "cancel", - "cancel": "cancel", - } - return mapping.get(text) - - @classmethod - def _build_duplicate_display_row( - cls, - item: Any, - *, - backend_name: str, - original_url: str, - ) -> Dict[str, Any]: - try: - extracted = build_display_row(item, keys=["title", "store", "hash", "ext", "size"]) - except Exception: - extracted = {} - - title = extracted.get("title") or cls._extract_duplicate_title(item) - store_name = extracted.get("store") or get_field(item, "store") or backend_name - file_hash = extracted.get("hash") or get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "" - ext_text = str(extracted.get("ext") or get_field(item, "ext") or "").strip() - size_raw = extracted.get("size") - if size_raw is None: - size_raw = get_field(item, "size_bytes") - if size_raw is None: - size_raw = get_field(item, "size") - - if not ext_text: - for candidate in (get_field(item, "path"), get_field(item, "title"), get_field(item, "name")): - candidate_text = str(candidate or "").strip() - if not candidate_text: - continue - suffix = Path(candidate_text).suffix.lstrip(".") - if suffix: - ext_text = suffix - break - - title_text = str(title) - tag_text = ", ".join(cls._extract_duplicate_namespace_tags(item)) - store_text = str(store_name or backend_name) - file_hash_text = str(file_hash or "") - selection_args = None - selection_action = None - selection_url = None - if file_hash_text and store_text and file_hash_text.strip().lower() != "unknown": - selection_args, selection_action = build_hash_store_selection( - file_hash_text, - store_text, - ) - if selection_args and len(selection_args) >= 2: - normalized_hash = str(selection_args[1]).split("hash:", 1)[-1].strip() - if normalized_hash: - file_hash_text = normalized_hash - selection_url = f"hydrus://{store_text}/{normalized_hash}" - - columns: List[tuple[str, Any]] = [("Title", title_text)] - if tag_text: - columns.append(("Tag", tag_text)) - columns.extend( - [ - ("Store", store_text), - ("Size", size_raw), - ("Ext", ext_text), - ("URL", original_url), - ] - ) - - metadata = dict(item) if isinstance(item, dict) else {} - if file_hash_text: - metadata.setdefault("hash", file_hash_text) - if store_text: - metadata.setdefault("store", store_text) - if ext_text: - metadata.setdefault("ext", ext_text) - if size_raw is not None: - metadata.setdefault("size", size_raw) - metadata.setdefault("size_bytes", size_raw) - metadata.setdefault("url", original_url) - if selection_url: - metadata.setdefault("selection_url", selection_url) - - payload = build_table_result_payload( - title=title_text, - columns=columns, - selection_args=selection_args, - selection_action=selection_action, - store=store_text, - hash=file_hash_text, - ext=ext_text, - size=size_raw, - size_bytes=size_raw, - url=original_url, - tags_flat=metadata.get("tags_flat"), - full_metadata=metadata, - ) - if selection_url: - payload["path"] = selection_url - payload["selection_url"] = selection_url - return payload - - @classmethod - def _fetch_duplicate_metadata_for_hash( - cls, - backend: Any, - *, - backend_name: str, - file_hash: str, - ) -> Dict[str, Any]: - metadata: Optional[Dict[str, Any]] = None - - fetcher = getattr(backend, "fetch_file_metadata", None) - if callable(fetcher): - try: - payload = fetcher(file_hash) - except TypeError: - try: - payload = fetcher(file_hash=file_hash) - except Exception: - payload = None - except Exception: - payload = None - - if isinstance(payload, dict): - meta_list = payload.get("metadata") - if isinstance(meta_list, list) and meta_list and isinstance(meta_list[0], dict): - metadata = dict(meta_list[0]) - else: - metadata = dict(payload) - - metadata = cls._enrich_duplicate_metadata( - metadata, - backend, - backend_name=backend_name, - file_hash=file_hash, - ) - - metadata.setdefault("hash", file_hash) - metadata.setdefault("store", backend_name) - return metadata - - @classmethod - def _enrich_duplicate_metadata( - cls, - metadata: Optional[Dict[str, Any]], - backend: Any, - *, - backend_name: str, - file_hash: str, - ) -> Dict[str, Any]: - result = dict(metadata) if isinstance(metadata, dict) else {} - - if result: - extractor = getattr(backend, "_extract_title_and_tags", None) - if callable(extractor): - file_id_value = get_field(result, "file_id") or file_hash - try: - extracted_title, extracted_tags = extractor(result, file_id_value) - except Exception: - extracted_title, extracted_tags = None, None - - if not get_field(result, "tags_flat") and isinstance(extracted_tags, SequenceABC) and not isinstance(extracted_tags, (str, bytes, bytearray, Mapping)): - deduped_tags: List[str] = [] - seen_tags: set[str] = set() - for raw_tag in extracted_tags: - tag_text = str(raw_tag or "").strip() - lowered = tag_text.lower() - if not tag_text or lowered in seen_tags: - continue - seen_tags.add(lowered) - deduped_tags.append(tag_text) - if deduped_tags: - result["tags_flat"] = deduped_tags - - title_text = str(extracted_title or "").strip() - generic_title = f"Hydrus File {file_id_value}".strip() - if title_text and title_text != generic_title: - result.setdefault("title", title_text) - - if not result: - getter = getattr(backend, "get_metadata", None) - if callable(getter): - try: - payload = getter(file_hash) - except Exception: - payload = None - if isinstance(payload, dict): - result = dict(payload) - - getter = getattr(backend, "get_metadata", None) - if callable(getter) and not cls._has_duplicate_title(result): - try: - getter_payload = getter(file_hash) - except Exception: - getter_payload = None - if isinstance(getter_payload, dict): - for key, value in getter_payload.items(): - current = result.get(key) - if current not in (None, "", [], {}, ()): - continue - if value in (None, "", [], {}, ()): - continue - result[key] = value - - return result - - @classmethod - def _fetch_duplicate_metadata_for_hashes( - cls, - backend: Any, - *, - backend_name: str, - file_hashes: Sequence[str], - ) -> Dict[str, Dict[str, Any]]: - normalized_hashes: List[str] = [] - seen_hashes: set[str] = set() - for raw_hash in file_hashes or []: - normalized_hash = sh.normalize_hash(str(raw_hash) if raw_hash is not None else None) - if not normalized_hash or normalized_hash in seen_hashes: - continue - seen_hashes.add(normalized_hash) - normalized_hashes.append(normalized_hash) - - if not normalized_hashes: - return {} - - metadata_by_hash: Dict[str, Dict[str, Any]] = {} - fetcher = getattr(backend, "fetch_files_metadata", None) - if callable(fetcher): - try: - payload = fetcher( - normalized_hashes, - include_service_keys_to_tags=True, - include_file_url=True, - include_duration=True, - include_size=True, - include_mime=True, - ) - except TypeError: - try: - payload = fetcher( - file_hashes=normalized_hashes, - include_service_keys_to_tags=True, - include_file_url=True, - include_duration=True, - include_size=True, - include_mime=True, - ) - except Exception: - payload = None - except Exception: - payload = None - - if isinstance(payload, dict): - meta_list = payload.get("metadata") - if isinstance(meta_list, list): - for entry in meta_list: - if not isinstance(entry, dict): - continue - entry_hash = sh.normalize_hash(str(entry.get("hash") or entry.get("hash_hex") or entry.get("file_hash") or "")) - if not entry_hash: - continue - metadata_by_hash[entry_hash] = cls._enrich_duplicate_metadata( - dict(entry), - backend, - backend_name=backend_name, - file_hash=entry_hash, - ) - - for normalized_hash in normalized_hashes: - metadata = metadata_by_hash.get(normalized_hash) - if metadata is None: - metadata = cls._fetch_duplicate_metadata_for_hash( - backend, - backend_name=backend_name, - file_hash=normalized_hash, - ) - metadata.setdefault("hash", normalized_hash) - metadata.setdefault("store", backend_name) - metadata_by_hash[normalized_hash] = metadata - - return metadata_by_hash - - @classmethod - def _collect_existing_url_match_refs_for_url( - cls, - storage: Any, - canonical_url: str, - *, - hydrus_available: bool, - config: Optional[Dict[str, Any]] = None, - ) -> List[Dict[str, Any]]: - if not canonical_url: - return [] - if not cls._supports_storage_duplicate_lookup(canonical_url): - return [] - - config_dict = config if isinstance(config, dict) else {} - refs: List[Dict[str, Any]] = [] - seen_pairs: set[tuple[str, str]] = set() - seen_backends: set[str] = set() - - def _append_ref(backend_name: str, backend: Any, *, item: Any = None, file_hash_hint: Optional[str] = None, is_exact: bool = False) -> None: - normalized_hash = sh.normalize_hash(str(file_hash_hint) if file_hash_hint is not None else None) - if not normalized_hash: - normalized_hash = cls._extract_hash_from_search_hit(item) - pair_key = (str(backend_name or "").strip().lower(), str(normalized_hash or "")) - if pair_key in seen_pairs: - return - seen_pairs.add(pair_key) - refs.append( - { - "backend_name": str(backend_name or "").strip(), - "backend": backend, - "hash": normalized_hash, - "item": dict(item) if isinstance(item, dict) else item, - "is_exact": bool(is_exact), - } - ) - - def _iter_backends() -> List[tuple[str, Any]]: - backends: List[tuple[str, Any]] = [] - if storage is not None: - try: - backend_names = list(storage.list_searchable_backends() or []) - except Exception: - backend_names = [] - - for backend_name in backend_names: - try: - backend = storage[backend_name] - except Exception: - continue - name_text = str(backend_name).strip() - if not name_text or name_text.lower() == "temp": - continue - key = name_text.lower() - if key in seen_backends: - continue - seen_backends.add(key) - backends.append((name_text, backend)) - - try: - registry_helpers = cls._load_provider_registry() - get_plugin = registry_helpers.get("get_plugin") - hydrus_provider = get_plugin("hydrusnetwork", config_dict) if callable(get_plugin) else None - if hydrus_provider is not None: - for backend_name, backend in hydrus_provider.iter_backends(): - name_text = str(backend_name or "").strip() - if not name_text: - continue - key = name_text.lower() - if key in seen_backends: - continue - seen_backends.add(key) - backends.append((name_text, backend)) - except Exception: - pass - - return backends - - for backend_name, backend in _iter_backends(): - try: - if not hydrus_available and str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": - continue - except Exception: - pass - - found_exact = False - lookup_exact = getattr(backend, "find_hashes_by_url", None) - if callable(lookup_exact): - try: - hashes = lookup_exact(canonical_url) or [] - except Exception: - hashes = [] - if isinstance(hashes, (list, tuple, set)): - for existing_hash in hashes: - normalized_hash = sh.normalize_hash(str(existing_hash) if existing_hash is not None else None) - if not normalized_hash: - continue - found_exact = True - _append_ref( - backend_name, - backend, - file_hash_hint=normalized_hash, - is_exact=True, - ) - if found_exact: - continue - - searcher = getattr(backend, "search", None) - if callable(searcher): - try: - hits = searcher(f"url:{canonical_url}", limit=5, minimal=True) or [] - except Exception: - hits = [] - for hit in hits: - _append_ref(backend_name, backend, item=hit) - - return refs - - @classmethod - def _find_existing_url_matches_for_url( - cls, - storage: Any, - canonical_url: str, - *, - hydrus_available: bool, - config: Optional[Dict[str, Any]] = None, - ) -> List[Dict[str, Any]]: - refs = cls._collect_existing_url_match_refs_for_url( - storage, - canonical_url, - hydrus_available=hydrus_available, - config=config, - ) - if not refs: - return [] - - matches: List[Dict[str, Any]] = [] - exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} - prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} - - for ref in refs: - if not ref.get("is_exact"): - continue - backend_name = str(ref.get("backend_name") or "").strip() - backend_key = backend_name.lower() - normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) - if not backend_key or not normalized_hash: - continue - bucket = exact_hashes_by_backend.setdefault( - backend_key, - { - "backend_name": backend_name, - "backend": ref.get("backend"), - "hashes": [], - }, - ) - if normalized_hash not in bucket["hashes"]: - bucket["hashes"].append(normalized_hash) - - for backend_key, bucket in exact_hashes_by_backend.items(): - metadata_map = cls._fetch_duplicate_metadata_for_hashes( - bucket.get("backend"), - backend_name=str(bucket.get("backend_name") or backend_key), - file_hashes=list(bucket.get("hashes") or []), - ) - for normalized_hash, metadata in metadata_map.items(): - prefetched_metadata[(backend_key, normalized_hash)] = metadata - - for ref in refs: - backend_name = str(ref.get("backend_name") or "").strip() - backend_key = backend_name.lower() - normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) - if ref.get("is_exact") and normalized_hash: - candidate = prefetched_metadata.get((backend_key, normalized_hash)) - if candidate is None: - candidate = cls._fetch_duplicate_metadata_for_hash( - ref.get("backend"), - backend_name=backend_name, - file_hash=normalized_hash, - ) - else: - item = ref.get("item") - candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} - - if normalized_hash: - candidate.setdefault("hash", normalized_hash) - candidate.setdefault("store", backend_name) - matches.append( - cls._build_duplicate_display_row( - candidate, - backend_name=backend_name, - original_url=canonical_url, - ) - ) - - return matches - - @classmethod - def _find_existing_hash_for_url( - cls, storage: Any, canonical_url: str, *, hydrus_available: bool - ) -> Optional[str]: - hashes = cls._find_existing_hashes_for_url( - storage, - canonical_url, - hydrus_available=hydrus_available, - config={}, - ) - return hashes[0] if hashes else None - - @staticmethod - def _init_storage(config: Dict[str, Any]) -> tuple[Any, bool]: - """Initialize store registry and determine whether a Hydrus backend is usable.""" - storage = None - try: - from PluginCore.backend_registry import BackendRegistry as _Store - - storage = _Store(config) - except Exception: - storage = None - - hydrus_available = False - try: - from plugins.hydrusnetwork import api as hydrus_api - - hydrus_available = bool(hydrus_api.is_hydrus_available(config)) - except Exception: - hydrus_available = False - - if storage is not None and not hydrus_available: - try: - backend_names = list(storage.list_backends() or []) - except Exception: - backend_names = [] - for backend_name in backend_names: - try: - backend = storage[backend_name] - except Exception: - continue - if str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": - hydrus_available = True - break - - return storage, hydrus_available - - @staticmethod - def _supports_storage_duplicate_lookup(raw_url: str) -> bool: - text = str(raw_url or "").strip() - if not text: - return False - - try: - parsed = urlparse(text) - except Exception: - parsed = None - - scheme = str(getattr(parsed, "scheme", "") or "").strip().lower() - return scheme in {"http", "https", "ftp", "ftps"} - - @staticmethod - def _filter_supported_urls(raw_urls: Sequence[str]) -> tuple[List[str], List[str]]: - """Split explicit URLs into supported and unsupported buckets.""" - supported: List[str] = [] - unsupported: List[str] = [] - for raw in raw_urls or []: - text = str(raw or "").strip() - if not text: - continue - low = text.lower() - if low.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:")): - supported.append(text) - else: - unsupported.append(text) - return supported, unsupported - - @staticmethod - def _canonicalize_url_for_storage( - *, - requested_url: str, - provider_name: Optional[str] = None, - provider_instance: Optional[str] = None, - provider_item: Optional[Any] = None, - ) -> str: - """Return the URL key used for duplicate preflight lookups.""" - return str(requested_url or "").strip() - - @staticmethod - def _preflight_url_duplicate( - *, - canonical_url: str, - storage: Any, - hydrus_available: bool, - final_output_dir: Path, - auto_continue_duplicates: bool = True, - force_prompt_in_pipeline: bool = False, - ) -> bool: - """Run duplicate URL preflight against configured storage backends.""" - if not canonical_url or storage is None: - return True - return not sh.check_url_exists_in_storage( - urls=[canonical_url], - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - auto_continue_duplicates=auto_continue_duplicates, - force_prompt_in_pipeline=force_prompt_in_pipeline, - ) - - @staticmethod - def _parse_clip_spec_to_ranges(clip_spec: Optional[str]) -> Optional[List[tuple[int, int]]]: - """Parse clip spec strings like '2m-2m20s,5m-6m'.""" - text = str(clip_spec or "").strip() - if not text: - return None - - def _parse_time(value: str) -> Optional[int]: - s = str(value or "").strip().lower() - if not s: - return None - try: - if ":" in s: - parts = [int(p) for p in s.split(":")] - if len(parts) == 2: - return (parts[0] * 60) + parts[1] - if len(parts) == 3: - return (parts[0] * 3600) + (parts[1] * 60) + parts[2] - return None - total = 0 - number = "" - units_seen = False - for ch in s: - if ch.isdigit(): - number += ch - continue - if ch in {"h", "m", "s"} and number: - units_seen = True - val = int(number) - if ch == "h": - total += val * 3600 - elif ch == "m": - total += val * 60 - else: - total += val - number = "" - continue - return None - if number: - total += int(number) - if total == 0 and units_seen: - return 0 - return total if total >= 0 else None - except Exception: - return None - - ranges: List[tuple[int, int]] = [] - for chunk in [c.strip() for c in text.split(",") if c.strip()]: - if "-" not in chunk: - return None - left, right = chunk.split("-", 1) - start = _parse_time(left) - end = _parse_time(right) - if start is None or end is None or end < start: - return None - ranges.append((start, end)) - return ranges or None - - def _download_supported_urls(self, **kwargs: Any) -> int: - """Download pre-validated streaming URLs (wrapper used by tests).""" - urls = list(kwargs.get("supported_url") or []) - storage = kwargs.get("storage") - hydrus_available = bool(kwargs.get("hydrus_available")) - final_output_dir = kwargs.get("final_output_dir") - skip_preflight = bool(kwargs.get("skip_per_url_preflight")) - - if not urls: - return 1 - - for requested_url in urls: - canonical = self._canonicalize_url_for_storage(requested_url=requested_url) - if skip_preflight: - continue - ok = self._preflight_url_duplicate( - canonical_url=canonical, - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=Path(final_output_dir) if final_output_dir else Path.cwd(), - ) - if not ok: - # Duplicate skip is non-fatal for the whole batch. - continue - - return 0 - - def _maybe_show_playlist_table(self, **kwargs: Any) -> bool: - """Compat hook used by tests; playlist table rendering is handled elsewhere.""" - return False - - def _maybe_show_format_table_for_single_url(self, **kwargs: Any) -> Optional[int]: - """Compat hook used by tests; format table rendering is handled elsewhere.""" - return None - - def _run_streaming_urls( - self, - *, - streaming_urls: Sequence[str], - args: Sequence[str], - config: Dict[str, Any], - parsed: Dict[str, Any], - ) -> int: - """Compat wrapper for tests that exercise legacy streaming dispatch flow.""" - storage, hydrus_available = self._init_storage(config) - supported_url, _unsupported = self._filter_supported_urls(streaming_urls) - if not supported_url: - return 1 - - final_output_dir = resolve_target_dir(parsed, config) - if final_output_dir is None: - return 1 - - query_text = str(parsed.get("query") or "") - clip_spec = None - for token in [t.strip() for t in query_text.split(",") if t.strip()]: - if token.lower().startswith("clip:"): - clip_spec = token.split(":", 1)[1].strip() - break - clip_ranges = self._parse_clip_spec_to_ranges(clip_spec) - - ytdlp_tool = YtDlpTool(config) if callable(YtDlpTool) else None - playlist_items = parsed.get("item") - - return self._download_supported_urls( - supported_url=supported_url, - ytdlp_tool=ytdlp_tool, - args=list(args), - config=config, - final_output_dir=final_output_dir, - mode="audio", - clip_spec=clip_spec, - clip_ranges=clip_ranges, - query_hash_override=None, - embed_chapters=False, - write_sub=False, - quiet_mode=bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False, - playlist_items=playlist_items, - ytdl_format=(ytdlp_tool.default_format("audio") if ytdlp_tool and hasattr(ytdlp_tool, "default_format") else "best"), - skip_per_url_preflight=False, - forced_single_format_id=None, - forced_single_format_for_batch=False, - formats_cache={}, - storage=storage, - hydrus_available=hydrus_available, - download_timeout_seconds=int(config.get("_pipeobject_timeout_seconds") or 300) if isinstance(config, dict) else 300, - ) - - @classmethod - def _find_existing_hashes_for_url( - cls, - storage: Any, - canonical_url: str, - *, - hydrus_available: bool, - config: Optional[Dict[str, Any]] = None, - ) -> List[str]: - refs = cls._collect_existing_url_match_refs_for_url( - storage, - canonical_url, - hydrus_available=hydrus_available, - config=config, - ) - hashes: List[str] = [] - seen_hashes: set[str] = set() - for ref in refs: - normalized = sh.normalize_hash(str(ref.get("hash") or "")) - if not normalized or normalized in seen_hashes: - continue - seen_hashes.add(normalized) - hashes.append(normalized) - return hashes - - def _preflight_explicit_url_duplicates( - self, - *, - raw_urls: Sequence[str], - config: Dict[str, Any], - ) -> tuple[List[str], Optional[int], int]: - """Return (urls_to_process, early_exit, skipped_count).""" - urls = [str(u or "").strip() for u in (raw_urls or []) if str(u or "").strip()] - if not urls: - return [], None, 0 - - if bool(config.get("_skip_url_preflight")): - return urls, None, 0 - - storage, hydrus_available = self._init_storage(config) - duplicate_refs: Dict[str, List[Dict[str, Any]]] = {} - exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} - for url in urls: - refs = self._collect_existing_url_match_refs_for_url( - storage, - url, - hydrus_available=hydrus_available, - config=config, - ) - if not refs: - continue - duplicate_refs[url] = refs - for ref in refs: - if not ref.get("is_exact"): - continue - backend_name = str(ref.get("backend_name") or "").strip() - backend_key = backend_name.lower() - normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) - if not backend_key or not normalized_hash: - continue - bucket = exact_hashes_by_backend.setdefault( - backend_key, - { - "backend_name": backend_name, - "backend": ref.get("backend"), - "hashes": [], - }, - ) - if normalized_hash not in bucket["hashes"]: - bucket["hashes"].append(normalized_hash) - - if not duplicate_refs: - return urls, None, 0 - - prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} - for backend_key, bucket in exact_hashes_by_backend.items(): - metadata_map = self._fetch_duplicate_metadata_for_hashes( - bucket.get("backend"), - backend_name=str(bucket.get("backend_name") or backend_key), - file_hashes=list(bucket.get("hashes") or []), - ) - for normalized_hash, metadata in metadata_map.items(): - prefetched_metadata[(backend_key, normalized_hash)] = metadata - - duplicates: Dict[str, List[Dict[str, Any]]] = {} - for url, refs in duplicate_refs.items(): - rows: List[Dict[str, Any]] = [] - for ref in refs: - backend_name = str(ref.get("backend_name") or "").strip() - backend_key = backend_name.lower() - normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) - if ref.get("is_exact") and normalized_hash: - candidate = prefetched_metadata.get((backend_key, normalized_hash)) - if candidate is None: - candidate = self._fetch_duplicate_metadata_for_hash( - ref.get("backend"), - backend_name=backend_name, - file_hash=normalized_hash, - ) - else: - item = ref.get("item") - candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} - - if normalized_hash: - candidate.setdefault("hash", normalized_hash) - candidate.setdefault("store", backend_name) - rows.append( - self._build_duplicate_display_row( - candidate, - backend_name=backend_name, - original_url=url, - ) - ) - if rows: - duplicates[url] = rows - - duplicate_count = len(duplicates) - total_count = len(urls) - try: - debug_panel( - "download-file duplicate preflight", - [ - ("total_urls", total_count), - ("duplicate_urls", duplicate_count), - ], - border_style="yellow", - ) - except Exception: - pass - - table = Table(f"Duplicate URLs detected ({duplicate_count}/{total_count})", max_columns=12) - table._interactive(False) - duplicate_rows: List[Dict[str, Any]] = [] - for _url, rows in duplicates.items(): - for row in rows: - payload = dict(row) if isinstance(row, dict) else {} - duplicate_rows.append(payload) - table.add_result(payload) - - try: - pipeline_context.set_last_result_table_overlay(table, duplicate_rows) - except Exception: - pass - - try: - stdin_interactive = bool(sys.stdin and sys.stdin.isatty()) - except Exception: - stdin_interactive = False - - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm: AbstractContextManager[Any] = nullcontext() - if callable(suspend): - try: - maybe_cm = suspend() - if maybe_cm is not None: - cm = maybe_cm # type: ignore[assignment] - except Exception: - cm = nullcontext() - - policy = "skip" - with cm: - console = get_stderr_console() - try: - console.print(table) - except Exception: - pass - setattr(table, "_rendered_by_cmdlet", True) - - if stdin_interactive: - while True: - try: - raw_policy = Prompt.ask( - "Duplicate URLs found. Action? [I]gnore/[S]kip/[C]ancel", - default="skip", - console=console, - ) - except (EOFError, KeyboardInterrupt): - policy = "cancel" - break - - normalized_policy = self._normalize_duplicate_preflight_policy(raw_policy) - if normalized_policy is not None: - policy = normalized_policy - break - - try: - console.print("Please select one of: I, S, C, ignore, skip, cancel") - except Exception: - pass - else: - # Safe default in non-interactive runs: avoid redownloading known duplicates. - policy = "skip" - - if policy == "cancel": - try: - pipeline_context.request_pipeline_stop(reason="duplicate-url cancelled", exit_code=0) - except Exception: - pass - return [], 0, 0 - - if policy == "ignore": - return urls, None, 0 - - filtered = [u for u in urls if u not in duplicates] - skipped = len(urls) - len(filtered) - if skipped: - try: - log(f"Skipped {skipped} duplicate URL(s); processing remaining {len(filtered)}.", file=sys.stderr) - except Exception: - pass - return filtered, None, skipped - - @staticmethod - def _resolve_provider_preflight_items( - provider: Any, - *, - url: str, - parsed: Dict[str, Any], - args: Sequence[str], - ) -> List[Dict[str, Any]]: - resolver = getattr(provider, "resolve_preflight_items", None) - if not callable(resolver): - return [] - - try: - items = resolver(url, parsed=parsed, args=list(args)) - except TypeError: - try: - items = resolver(url) - except Exception: - items = None - except Exception: - items = None - - if not isinstance(items, list): - return [] - - normalized: List[Dict[str, Any]] = [] - for idx, item in enumerate(items, 1): - if not isinstance(item, dict): - continue - item_url = str(item.get("url") or "").strip() - if not item_url: - continue - playlist_index = item.get("playlist_index") - try: - playlist_index_value = int(playlist_index) - except Exception: - playlist_index_value = idx - normalized.append( - { - "url": item_url, - "playlist_index": playlist_index_value, - } - ) - - return normalized - - @staticmethod - def _build_provider_playlist_item_selector( - items: Sequence[Dict[str, Any]], - *, - remaining_urls: Sequence[str], - ) -> Optional[str]: - allowed_urls = { - str(url or "").strip() for url in (remaining_urls or []) if str(url or "").strip() - } - if not allowed_urls: - return None - - selectors: List[str] = [] - for idx, item in enumerate(items, 1): - item_url = str(item.get("url") or "").strip() - if not item_url or item_url not in allowed_urls: - continue - playlist_index = item.get("playlist_index") - try: - playlist_index_value = int(playlist_index) - except Exception: - playlist_index_value = idx - if playlist_index_value <= 0: - continue - selectors.append(str(playlist_index_value)) - - if not selectors: - return None - return ",".join(selectors) - - @staticmethod - def _format_timecode(seconds: int, *, force_hours: bool) -> str: - total = max(0, int(seconds)) - minutes, secs = divmod(total, 60) - hours, minutes = divmod(minutes, 60) - if force_hours: - return f"{hours:02d}:{minutes:02d}:{secs:02d}" - return f"{minutes:02d}:{secs:02d}" - - @staticmethod - def _rebase_subtitle_timestamp_text(text: str, offset_seconds: int) -> str: - if not text: - return text - - try: - offset_value = float(offset_seconds) - except Exception: - return text - - if offset_value <= 0: - return text - - timestamp_re = re.compile(r"(?<!\d)(?P<ts>(?:\d{2}:)?\d{2}:\d{2}(?:[\.,]\d{1,3})?)(?!\d)") - - def _shift(match: re.Match[str]) -> str: - original = str(match.group("ts") or "") - if not original: - return original - - frac_sep = "." - frac_digits = 0 - base = original - frac_seconds = 0.0 - if "." in original: - base, frac = original.split(".", 1) - frac_sep = "." - frac_digits = len(frac) - try: - frac_seconds = float(f"0.{frac}") if frac else 0.0 - except Exception: - frac_seconds = 0.0 - elif "," in original: - base, frac = original.split(",", 1) - frac_sep = "," - frac_digits = len(frac) - try: - frac_seconds = float(f"0.{frac}") if frac else 0.0 - except Exception: - frac_seconds = 0.0 - - parts = base.split(":") - if len(parts) == 3: - hours_s, minutes_s, seconds_s = parts - include_hours = True - elif len(parts) == 2: - hours_s = "0" - minutes_s, seconds_s = parts - include_hours = False - else: - return original - - try: - total = ( - (int(hours_s) * 3600) - + (int(minutes_s) * 60) - + int(seconds_s) - + frac_seconds - + offset_value - ) - except Exception: - return original - - total = max(0.0, total) - whole_seconds = int(total) - fraction = total - whole_seconds - hours, remainder = divmod(whole_seconds, 3600) - minutes, seconds = divmod(remainder, 60) - - if frac_digits > 0: - scale = 10 ** frac_digits - frac_value = int(round(fraction * scale)) - if frac_value >= scale: - frac_value = 0 - seconds += 1 - if seconds >= 60: - seconds = 0 - minutes += 1 - if minutes >= 60: - minutes = 0 - hours += 1 - frac_text = f"{frac_value:0{frac_digits}d}" - if include_hours or hours > 0: - return f"{hours:02d}:{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" - return f"{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" - - if include_hours or hours > 0: - return f"{hours:02d}:{minutes:02d}:{seconds:02d}" - return f"{minutes:02d}:{seconds:02d}" - - try: - return timestamp_re.sub(_shift, str(text)) - except Exception: - return text - - @classmethod - def _format_clip_range(cls, start_s: int, end_s: int) -> str: - force_hours = bool(start_s >= 3600 or end_s >= 3600) - return f"{cls._format_timecode(start_s, force_hours=force_hours)}-{cls._format_timecode(end_s, force_hours=force_hours)}" - - @classmethod - def _apply_clip_decorations( - cls, pipe_objects: List[Dict[str, Any]], clip_ranges: List[tuple[int, int]], *, source_king_hash: Optional[str] - ) -> None: - if not pipe_objects or len(pipe_objects) != len(clip_ranges): - return - - for po, (start_s, end_s) in zip(pipe_objects, clip_ranges): - clip_range = cls._format_clip_range(start_s, end_s) - clip_tag = f"clip:{clip_range}" - - po["title"] = clip_tag - - tags = po.get("tag") - if not isinstance(tags, list): - tags = [] - - tags = [t for t in tags if not str(t).strip().lower().startswith("title:")] - tags = [t for t in tags if not str(t).strip().lower().startswith("relationship:")] - tags.insert(0, f"title:{clip_tag}") - - if clip_tag not in tags: - tags.append(clip_tag) - - po["tag"] = tags - - notes = po.get("notes") - if isinstance(notes, dict): - sub_text = notes.get("sub") - if isinstance(sub_text, str) and sub_text.strip(): - notes["sub"] = cls._rebase_subtitle_timestamp_text(sub_text, start_s) - po["notes"] = notes - - if len(pipe_objects) < 2: - return - - hashes: List[str] = [] - for po in pipe_objects: - h_val = sh.normalize_hash(str(po.get("hash") or "")) - hashes.append(h_val or "") - - king_hash = sh.normalize_hash(source_king_hash) if source_king_hash else None - if not king_hash: - king_hash = hashes[0] if hashes and hashes[0] else None - if not king_hash: - return - - alt_hashes: List[str] = [h for h in hashes if h and h != king_hash] - if not alt_hashes: - return - - for po in pipe_objects: - po["relationships"] = {"king": [king_hash], "alt": list(alt_hashes)} - - def _run_impl( - self, - result: Any, - args: Sequence[str], - config: Dict[str, Any] - ) -> int: - """Main download implementation for direct HTTP files.""" - progress = PipelineProgress(pipeline_context) - prev_progress = None - had_progress_key = False - try: - # Allow providers to tap into the active PipelineProgress (optional). - try: - if isinstance(config, dict): - had_progress_key = "_pipeline_progress" in config - prev_progress = config.get("_pipeline_progress") - config["_pipeline_progress"] = progress - except Exception: - pass - - # Parse arguments - parsed = parse_cmdlet_args(args, self) - registry = self._load_provider_registry() - selection_url_prefixes = self._selection_url_prefixes(registry) - explicit_input = parsed.get("url") - - # Resolve URLs from -url or positional arguments - url_candidates = parsed.get("url") or [ - a for a in parsed.get("args", []) - if isinstance(a, str) and ( - a.startswith("http") or "://" in a or ":" in a - or "🧲" in a - and not a.startswith("-") - ) - ] - from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup - raw_url = normalize_url_list(url_candidates) - local_source_inputs: List[str] = [] - if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input): - local_source_inputs = [str(explicit_input)] - - quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False - - # Fallback to piped items if no explicit URLs provided - piped_items = [] - if not raw_url: - piped_items = sh.normalize_result_items( - result, - include_falsey_single=True, - ) - - # Handle TABLE_AUTO_STAGES routing: if a piped item has _selection_args, - # re-invoke download-file with those args instead of processing the PipeObject itself. - if piped_items and not raw_url: - selection_runs: List[List[str]] = [] - residual_items: List[Any] = [] - - for item in piped_items: - handled = False - try: - normalized_args, _normalized_action, item_url = extract_selection_fields( - item, - extra_url_prefixes=selection_url_prefixes, - ) - if normalized_args: - if selection_args_have_url( - normalized_args, - extra_url_prefixes=selection_url_prefixes, - ): - selection_runs.append(list(normalized_args)) - handled = True - elif item_url: - selection_runs.append([str(item_url)] + list(normalized_args)) - handled = True - except Exception as e: - debug_panel( - "download-file selection args failed", - [("error", e)], - border_style="yellow", - ) - handled = False - if not handled: - residual_items.append(item) - if selection_runs: - selection_urls: List[str] = [] - - for run_args in selection_runs: - for u in extract_urls_from_selection_args( - run_args, - extra_url_prefixes=selection_url_prefixes, - ): - if u not in selection_urls: - selection_urls.append(u) - - original_skip_preflight = None - original_timeout = None - original_skip_direct = None - original_batch_total = None - original_batch_index = None - original_batch_label = None - original_suppress_nested = None - try: - if isinstance(config, dict): - original_skip_preflight = config.get("_skip_url_preflight") - original_timeout = config.get("_pipeobject_timeout_seconds") - original_skip_direct = config.get("_skip_direct_on_streaming_failure") - original_batch_total = config.get("_download_file_batch_total") - original_batch_index = config.get("_download_file_batch_index") - original_batch_label = config.get("_download_file_batch_label") - original_suppress_nested = config.get("_download_file_suppress_nested_pipe_progress") - except Exception: - original_skip_preflight = None - original_timeout = None - original_batch_total = None - original_batch_index = None - original_batch_label = None - original_suppress_nested = None - - try: - if selection_urls: - # Skip Duplicate Preflight on selection re-entry: - # User has already seen the table/status and explicitly selected an item. - # Skipping this reduces DB load and latency. - if isinstance(config, dict): - config["_skip_url_preflight"] = True - config["_skip_direct_on_streaming_failure"] = True - - if isinstance(config, dict) and config.get("_pipeobject_timeout_seconds") is None: - # Use a generous default for individual items - config["_pipeobject_timeout_seconds"] = 600 - - successes = 0 - failures = 0 - last_code = 0 - total_selection = len(selection_runs) - preview_items = list(selection_urls[:5]) or [ - self._selection_run_label( - run_args, - extra_url_prefixes=selection_url_prefixes, - ) - for run_args in selection_runs[:5] - ] - try: - progress.ensure_local_ui( - label="download-file", - total_items=total_selection, - items_preview=preview_items, - ) - except Exception: - pass - try: - progress.begin_pipe( - total_items=total_selection, - items_preview=preview_items, - ) - except Exception: - pass - for idx, run_args in enumerate(selection_runs, 1): - run_label = self._selection_run_label( - run_args, - extra_url_prefixes=selection_url_prefixes, - ) - try: - progress.set_status( - f"downloading {idx}/{total_selection}: {run_label}" - ) - except Exception: - pass - try: - if isinstance(config, dict): - config["_download_file_batch_total"] = total_selection - config["_download_file_batch_index"] = idx - config["_download_file_batch_label"] = run_label - config["_download_file_suppress_nested_pipe_progress"] = True - except Exception: - pass - exit_code = self._run_impl(None, run_args, config) - if exit_code == 0: - successes += 1 - else: - failures += 1 - last_code = exit_code - - piped_items = residual_items - if not piped_items: - if successes > 0: - return 0 - return last_code or 1 - finally: - try: - if isinstance(config, dict): - if original_skip_preflight is None: - config.pop("_skip_url_preflight", None) - else: - config["_skip_url_preflight"] = original_skip_preflight - if original_timeout is None: - config.pop("_pipeobject_timeout_seconds", None) - else: - config["_pipeobject_timeout_seconds"] = original_timeout - if original_skip_direct is None: - config.pop("_skip_direct_on_streaming_failure", None) - else: - config["_skip_direct_on_streaming_failure"] = original_skip_direct - if original_batch_total is None: - config.pop("_download_file_batch_total", None) - else: - config["_download_file_batch_total"] = original_batch_total - if original_batch_index is None: - config.pop("_download_file_batch_index", None) - else: - config["_download_file_batch_index"] = original_batch_index - if original_batch_label is None: - config.pop("_download_file_batch_label", None) - else: - config["_download_file_batch_label"] = original_batch_label - if original_suppress_nested is None: - config.pop("_download_file_suppress_nested_pipe_progress", None) - else: - config["_download_file_suppress_nested_pipe_progress"] = original_suppress_nested - except Exception: - pass - - had_piped_input = False - try: - if isinstance(result, list): - had_piped_input = bool(result) - else: - had_piped_input = bool(result) - except Exception: - had_piped_input = False - - # UX: In piped mode, allow a single positional arg to be the destination directory. - # Example: @1-4 | download-file "C:\\Users\\Me\\Downloads\\yoyo" - if (had_piped_input and raw_url and len(raw_url) == 1 - and (not parsed.get("path"))): - candidate = str(raw_url[0] or "").strip() - low = candidate.lower() - looks_like_url = low.startswith( - ("http://", "https://", "ftp://", "magnet:", "torrent:") - + tuple(selection_url_prefixes) - ) - looks_like_provider = ( - ":" in candidate and not candidate.startswith( - ("http:", "https:", "ftp:", "ftps:", "file:") - + tuple(selection_url_prefixes) - ) - ) - looks_like_windows_path = ( - (len(candidate) >= 2 and candidate[1] == ":") - or candidate.startswith("\\\\") or candidate.startswith("\\") - or candidate.endswith(("\\", - "/")) - ) - if (not looks_like_url) and ( - not looks_like_provider) and looks_like_windows_path: - parsed["path"] = candidate - raw_url = [] - piped_items = self._collect_piped_items_if_no_urls(result, raw_url) - - if not raw_url and not piped_items and not local_source_inputs: - log("No url or piped items to download", file=sys.stderr) - return 1 - - # Provider-pre-check (e.g. Internet Archive format picker) - picker_result = self._maybe_show_provider_picker( - raw_urls=raw_url, - piped_items=piped_items, - parsed=parsed, - config=config, - registry=registry, - ) - if picker_result is not None: - return int(picker_result) - - # Re-check picker if partial processing occurred - picker_result = self._maybe_show_provider_picker( - raw_urls=raw_url, - piped_items=piped_items, - parsed=parsed, - config=config, - registry=registry, - ) - if picker_result is not None: - return int(picker_result) - - # Get output directory - final_output_dir = resolve_target_dir(parsed, config) - if not final_output_dir: - return 1 - - try: - debug_panel( - "download-file plan", - [ - ("output_dir", final_output_dir), - ("remaining_urls", len(raw_url)), - ("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))), - ], - border_style="cyan", - ) - except Exception: - pass - - # If the caller isn't running the shared pipeline Live progress UI (e.g. direct - # cmdlet execution), start a minimal local pipeline progress panel so downloads - # show consistent, Rich-formatted progress (like download-media). - total_items = max(1, len(raw_url or []) + len(piped_items or [])) - preview = build_pipeline_preview(raw_url, piped_items) - - progress.ensure_local_ui( - label="download-file", - total_items=total_items, - items_preview=preview - ) - - raw_url, preflight_exit, skipped_dupe_count = self._preflight_explicit_url_duplicates( - raw_urls=raw_url, - config=config, - ) - if preflight_exit is not None: - return int(preflight_exit) - - downloaded_count = 0 - - if local_source_inputs: - downloaded_count += self._process_explicit_local_sources( - local_sources=local_source_inputs, - final_output_dir=final_output_dir, - parsed=parsed, - progress=progress, - config=config, - ) - - storage_downloaded, piped_items, storage_exit = self._process_storage_items( - piped_items=piped_items, - parsed=parsed, - config=config, - final_output_dir=final_output_dir, - ) - downloaded_count += int(storage_downloaded) - if storage_exit is not None: - return int(storage_exit) - - if skipped_dupe_count and not raw_url and not piped_items: - return 0 if downloaded_count > 0 else 0 - - urls_downloaded, early_exit = self._process_explicit_urls( - raw_urls=raw_url, - final_output_dir=final_output_dir, - config=config, - quiet_mode=quiet_mode, - registry=registry, - progress=progress, - parsed=parsed, - args=args, - context_items=(result if isinstance(result, list) else ([result] if result else [])), - ) - downloaded_count += int(urls_downloaded) - if early_exit is not None: - return int(early_exit) - - provider_downloaded, magnet_submissions = self._process_provider_items( - piped_items=piped_items, - final_output_dir=final_output_dir, - config=config, - quiet_mode=quiet_mode, - registry=registry, - progress=progress, - ) - downloaded_count += provider_downloaded - - if downloaded_count > 0 or magnet_submissions > 0: - # Render detail panels for downloaded items when download-file is the last stage. - self._maybe_render_download_details(config=config) - return 0 - - return 1 - - except Exception as e: - log(f"Error in download-file: {e}", file=sys.stderr) - return 1 - - finally: - try: - if isinstance(config, dict): - if had_progress_key: - config["_pipeline_progress"] = prev_progress - else: - config.pop("_pipeline_progress", None) - except Exception: - pass - progress.close_local_ui(force_complete=True) - - def _maybe_show_provider_picker( - self, - *, - raw_urls: Sequence[str], - piped_items: Sequence[Any], - parsed: Dict[str, Any], - config: Dict[str, Any], - registry: Dict[str, Any], - ) -> Optional[int]: - """Generic hook for providers to show a selection table (e.g. Internet Archive format picker).""" - total_inputs = len(raw_urls or []) + len(piped_items or []) - if total_inputs != 1: - return None - - target_url = None - if raw_urls: - target_url = str(raw_urls[0]) - elif piped_items: - target_url = str(get_field(piped_items[0], "path") or get_field(piped_items[0], "url") or "") - - if not target_url: - return None - - match_provider_name_for_url = registry.get("match_plugin_name_for_url") - get_provider = registry.get("get_plugin") - - provider_name = None - if match_provider_name_for_url: - try: - provider_name = match_provider_name_for_url(target_url) - except Exception: - pass - - if provider_name and get_provider: - provider = get_provider(provider_name, config) - if provider and hasattr(provider, "maybe_show_picker"): - try: - quiet_mode = bool(config.get("_quiet_background_output")) - res = provider.maybe_show_picker( - url=target_url, - item=piped_items[0] if piped_items else None, - parsed=parsed, - config=config, - quiet_mode=quiet_mode, - ) - if res is not None: - return int(res) - except Exception as e: - debug_panel( - "download-file picker error", - [ - ("plugin", provider_name), - ("url", target_url), - ("error", e), - ], - border_style="yellow", - ) - - return None - - -# Module-level singleton registration -CMDLET = Download_File() +from .download_storage import ( + _iter_storage_export_refs, + _export_store_file, + _process_storage_items, + _process_explicit_local_sources, + _extract_hash_from_search_hit, + _iter_duplicate_tag_values, + _extract_duplicate_namespace_tags, + _extract_duplicate_title_tag, + _extract_duplicate_title, + _has_duplicate_title, + _build_duplicate_display_row, + _fetch_duplicate_metadata_for_hash, + _enrich_duplicate_metadata, + _fetch_duplicate_metadata_for_hashes, + _collect_existing_url_match_refs_for_url, + _find_existing_url_matches_for_url, + _find_existing_hash_for_url, + _find_existing_hashes_for_url, + _init_storage, + _supports_storage_duplicate_lookup, + _filter_supported_urls, + _canonicalize_url_for_storage, + _preflight_url_duplicate, + _preflight_explicit_url_duplicates, + _download_supported_urls, + _maybe_show_playlist_table, + _maybe_show_format_table_for_single_url, + _run_streaming_urls, +) diff --git a/cmdlet/file/download_core.py b/cmdlet/file/download_core.py new file mode 100644 index 0000000..ca55147 --- /dev/null +++ b/cmdlet/file/download_core.py @@ -0,0 +1,897 @@ +"""Generic file/stream downloader — core orchestration. + +Supports: +- Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp) +- Piped plugin items (uses plugin.download when available) +- Streaming sites via yt-dlp (YouTube, Bandcamp, etc.) +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence as SequenceABC +import sys +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence +from urllib.parse import urlparse +from contextlib import AbstractContextManager, nullcontext +import shutil +import webbrowser + +from API.HTTP import download_direct_file +from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult +from SYS.logger import log, debug_panel, is_debug_enabled +from SYS.payload_builders import build_file_result_payload, build_table_result_payload +from SYS.pipeline_progress import PipelineProgress +from SYS.result_table import Table, build_display_row +from SYS.rich_display import stderr_console as get_stderr_console +from SYS import pipeline as pipeline_context +from SYS.item_accessors import get_result_title +from rich.prompt import Prompt +from SYS.selection_builder import ( + build_hash_store_selection, + extract_selection_fields, + extract_urls_from_selection_args, + selection_args_have_url, +) +from SYS.utils import sha256_file + +try: + from plugins.ytdlp import YtDlpTool # type: ignore +except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers + YtDlpTool = None # type: ignore + +from .. import _shared as sh + +Cmdlet = sh.Cmdlet +CmdletArg = sh.CmdletArg +SharedArgs = sh.SharedArgs +QueryArg = sh.QueryArg +parse_cmdlet_args = sh.parse_cmdlet_args +register_url_with_local_library = sh.register_url_with_local_library +coerce_to_pipe_object = sh.coerce_to_pipe_object +get_field = sh.get_field +resolve_target_dir = sh.resolve_target_dir +coerce_to_path = sh.coerce_to_path +build_pipeline_preview = sh.build_pipeline_preview + + +class Download_File(Cmdlet): + """Class-based download-file cmdlet - direct HTTP downloads.""" + + def __init__(self) -> None: + """Initialize download-file cmdlet.""" + super().__init__( + name="download-file", + summary="Download files or streaming media", + usage= + "download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]", + alias=["dl-file", + "download-http"], + arg=[ + SharedArgs.URL, + SharedArgs.PLUGIN, + SharedArgs.INSTANCE, + SharedArgs.PATH, + SharedArgs.QUERY, + CmdletArg( + name="name", + type="string", + description="Output filename override for store exports.", + ), + CmdletArg( + name="browser", + type="flag", + description="Open a backend-provided browser URL instead of exporting to disk when available.", + ), + QueryArg( + "clip", + key="clip", + aliases=["range", + "section", + "sections"], + type="string", + required=False, + description=( + "Clip time ranges via -query keyed fields (e.g. clip:1m-2m or clip:00:01-00:10). " + "Comma-separated values supported." + ), + query_only=True, + ), + CmdletArg( + name="item", + type="string", + description="Item selection for playlists/formats", + ), + ], + detail=[ + "Download files directly via HTTP or streaming media via yt-dlp.", + "Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.", + "Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.", + "For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.", + ], + exec=self.run, + ) + self.register() + + def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: + """Main execution method.""" + try: + debug_panel( + "download-file", + [ + ("args", list(args)), + ("has_piped_input", bool(result)), + ], + border_style="cyan", + ) + except Exception: + pass + return self._run_impl(result, args, config) + + @staticmethod + def _path_from_download_result(result_obj: Any) -> Path: + """Normalize downloader return values to a concrete filesystem path.""" + resolved = coerce_to_path(result_obj) + if resolved is None: + raise DownloadError("Could not determine downloaded file path") + return resolved + + @staticmethod + def _selection_run_label( + run_args: Sequence[str], + *, + extra_url_prefixes: Sequence[str] = (), + ) -> str: + try: + urls = extract_urls_from_selection_args( + run_args, + extra_url_prefixes=extra_url_prefixes, + ) + if urls: + return str(urls[0]) + except Exception: + pass + + for arg in run_args: + text = str(arg or "").strip() + if text and not text.startswith("-"): + return text + return "item" + + @staticmethod + def _batch_progress_state(config: Optional[Dict[str, Any]]) -> tuple[bool, int, int, str]: + if not isinstance(config, dict): + return False, 0, 0, "" + + suppress_nested = bool(config.get("_download_file_suppress_nested_pipe_progress")) + if not suppress_nested: + return False, 0, 0, "" + + try: + total = max(0, int(config.get("_download_file_batch_total") or 0)) + except Exception: + total = 0 + try: + index = max(0, int(config.get("_download_file_batch_index") or 0)) + except Exception: + index = 0 + try: + label = str(config.get("_download_file_batch_label") or "").strip() + except Exception: + label = "" + + return True, total, index, label + + @staticmethod + def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]: + loader = registry.get("list_selection_url_prefixes") + if not callable(loader): + return [] + try: + values = loader() or [] + except Exception: + return [] + return [str(value).strip().lower() for value in values if str(value or "").strip()] + + def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]: + if value is None: + return None + try: + normalized = str(value).strip() + except Exception: + return None + if not normalized: + return None + if "." in normalized: + normalized = normalized.split(".", 1)[0] + return normalized.lower() + + def _provider_key_from_item(self, item: Any) -> Optional[str]: + table_hint = get_field(item, "table") + key = self._normalize_provider_key(table_hint) + if key: + return key + provider_hint = get_field(item, "plugin") + key = self._normalize_provider_key(provider_hint) + if key: + return key + return self._normalize_provider_key(get_field(item, "source")) + + @staticmethod + def _path_looks_local(value: Any) -> bool: + text = str(value or "").strip() + if not text: + return False + if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")): + return False + if len(text) >= 2 and text[1] == ":": + return True + if text.startswith(("\\", "/", ".", "~")): + return True + return Path(text).exists() + + @staticmethod + def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str: + candidates = [ + get_result_title(result, "title", "name", "filename"), + get_result_title(metadata or {}, "title", "name", "filename"), + ] + for candidate in candidates: + if candidate is None: + continue + text = str(candidate).strip() + if text: + return text + return "" + + @staticmethod + def _sanitize_export_filename(name: str) -> str: + allowed_chars: List[str] = [] + for ch in str(name or ""): + if ch.isalnum() or ch in {"-", "_", " ", "."}: + allowed_chars.append(ch) + else: + allowed_chars.append(" ") + sanitized = " ".join("".join(allowed_chars).split()) + return sanitized or "export" + + @staticmethod + def _unique_export_path(path: Path) -> Path: + if not path.exists(): + return path + stem = path.stem + suffix = path.suffix + parent = path.parent + counter = 1 + while True: + candidate = parent / f"{stem} ({counter}){suffix}" + if not candidate.exists(): + return candidate + counter += 1 + + @staticmethod + def _load_provider_registry() -> Dict[str, Any]: + """Lightweight accessor for plugin helpers without hard dependencies.""" + try: + from PluginCore import registry as provider_registry # type: ignore + from PluginCore.base import SearchResult # type: ignore + + return { + "get_plugin": getattr(provider_registry, "get_plugin", None), + "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None), + "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None), + "SearchResult": SearchResult, + } + except Exception: + return { + "get_plugin": None, + "match_plugin_name_for_url": None, + "list_selection_url_prefixes": None, + "SearchResult": None, + } + + @classmethod + def _normalize_duplicate_preflight_policy(cls, value: Any) -> Optional[str]: + text = str(value or "").strip().lower() + if not text: + return "skip" + mapping = { + "i": "ignore", + "ignore": "ignore", + "s": "skip", + "skip": "skip", + "c": "cancel", + "cancel": "cancel", + } + return mapping.get(text) + + def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None: + try: + stage_ctx = pipeline_context.get_stage_context() + except Exception: + stage_ctx = None + + is_last_stage = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if not is_last_stage: + return + + try: + quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False + except Exception: + quiet_mode = False + if quiet_mode: + return + + emitted_items: List[Any] = [] + try: + emitted_items = list(getattr(stage_ctx, "emits", None) or []) if stage_ctx is not None else [] + except Exception: + emitted_items = [] + + if not emitted_items: + return + + try: + live_progress = pipeline_context.get_live_progress() + except Exception: + live_progress = None + + if live_progress is not None: + try: + pipe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None + if isinstance(pipe_idx, int): + live_progress.finish_pipe(int(pipe_idx), force_complete=True) + except Exception: + pass + try: + live_progress.stop() + except Exception: + pass + try: + if hasattr(pipeline_context, "set_live_progress"): + pipeline_context.set_live_progress(None) + except Exception: + pass + + try: + subject = emitted_items[0] if len(emitted_items) == 1 else list(emitted_items) + from .._shared import display_and_persist_items + display_and_persist_items(list(emitted_items), title="Result", subject=subject) + except Exception: + pass + + try: + if stage_ctx is not None: + stage_ctx.emits = [] + except Exception: + pass + + def _maybe_show_provider_picker( + self, + *, + raw_urls: Sequence[str], + piped_items: Sequence[Any], + parsed: Dict[str, Any], + config: Dict[str, Any], + registry: Dict[str, Any], + ) -> Optional[int]: + """Generic hook for providers to show a selection table (e.g. Internet Archive format picker).""" + total_inputs = len(raw_urls or []) + len(piped_items or []) + if total_inputs != 1: + return None + + target_url = None + if raw_urls: + target_url = str(raw_urls[0]) + elif piped_items: + target_url = str(get_field(piped_items[0], "path") or get_field(piped_items[0], "url") or "") + + if not target_url: + return None + + match_provider_name_for_url = registry.get("match_plugin_name_for_url") + get_provider = registry.get("get_plugin") + + provider_name = None + if match_provider_name_for_url: + try: + provider_name = match_provider_name_for_url(target_url) + except Exception: + pass + + if provider_name and get_provider: + provider = get_provider(provider_name, config) + if provider and hasattr(provider, "maybe_show_picker"): + try: + quiet_mode = bool(config.get("_quiet_background_output")) + res = provider.maybe_show_picker( + url=target_url, + item=piped_items[0] if piped_items else None, + parsed=parsed, + config=config, + quiet_mode=quiet_mode, + ) + if res is not None: + return int(res) + except Exception as e: + debug_panel( + "download-file picker error", + [ + ("plugin", provider_name), + ("url", target_url), + ("error", e), + ], + border_style="yellow", + ) + + return None + + def _run_impl( + self, + result: Any, + args: Sequence[str], + config: Dict[str, Any] + ) -> int: + """Main download implementation for direct HTTP files.""" + progress = PipelineProgress(pipeline_context) + prev_progress = None + had_progress_key = False + try: + try: + if isinstance(config, dict): + had_progress_key = "_pipeline_progress" in config + prev_progress = config.get("_pipeline_progress") + config["_pipeline_progress"] = progress + except Exception: + pass + + parsed = parse_cmdlet_args(args, self) + registry = self._load_provider_registry() + selection_url_prefixes = self._selection_url_prefixes(registry) + explicit_input = parsed.get("url") + + url_candidates = parsed.get("url") or [ + a for a in parsed.get("args", []) + if isinstance(a, str) and ( + a.startswith("http") or "://" in a or ":" in a + or "\U0001f9f2" in a + and not a.startswith("-") + ) + ] + from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup + raw_url = normalize_url_list(url_candidates) + local_source_inputs: List[str] = [] + if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input): + local_source_inputs = [str(explicit_input)] + + quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False + + piped_items = [] + if not raw_url: + piped_items = sh.normalize_result_items( + result, + include_falsey_single=True, + ) + + if piped_items and not raw_url: + selection_runs: List[List[str]] = [] + residual_items: List[Any] = [] + + for item in piped_items: + handled = False + try: + normalized_args, _normalized_action, item_url = extract_selection_fields( + item, + extra_url_prefixes=selection_url_prefixes, + ) + if normalized_args: + if selection_args_have_url( + normalized_args, + extra_url_prefixes=selection_url_prefixes, + ): + selection_runs.append(list(normalized_args)) + handled = True + elif item_url: + selection_runs.append([str(item_url)] + list(normalized_args)) + handled = True + except Exception as e: + debug_panel( + "download-file selection args failed", + [("error", e)], + border_style="yellow", + ) + handled = False + if not handled: + residual_items.append(item) + if selection_runs: + selection_urls: List[str] = [] + + for run_args in selection_runs: + for u in extract_urls_from_selection_args( + run_args, + extra_url_prefixes=selection_url_prefixes, + ): + if u not in selection_urls: + selection_urls.append(u) + + original_skip_preflight = None + original_timeout = None + original_skip_direct = None + original_batch_total = None + original_batch_index = None + original_batch_label = None + original_suppress_nested = None + try: + if isinstance(config, dict): + original_skip_preflight = config.get("_skip_url_preflight") + original_timeout = config.get("_pipeobject_timeout_seconds") + original_skip_direct = config.get("_skip_direct_on_streaming_failure") + original_batch_total = config.get("_download_file_batch_total") + original_batch_index = config.get("_download_file_batch_index") + original_batch_label = config.get("_download_file_batch_label") + original_suppress_nested = config.get("_download_file_suppress_nested_pipe_progress") + except Exception: + original_skip_preflight = None + original_timeout = None + original_batch_total = None + original_batch_index = None + original_batch_label = None + original_suppress_nested = None + + try: + if selection_urls: + if isinstance(config, dict): + config["_skip_url_preflight"] = True + config["_skip_direct_on_streaming_failure"] = True + + if isinstance(config, dict) and config.get("_pipeobject_timeout_seconds") is None: + config["_pipeobject_timeout_seconds"] = 600 + + successes = 0 + failures = 0 + last_code = 0 + total_selection = len(selection_runs) + preview_items = list(selection_urls[:5]) or [ + self._selection_run_label( + run_args, + extra_url_prefixes=selection_url_prefixes, + ) + for run_args in selection_runs[:5] + ] + try: + progress.ensure_local_ui( + label="download-file", + total_items=total_selection, + items_preview=preview_items, + ) + except Exception: + pass + try: + progress.begin_pipe( + total_items=total_selection, + items_preview=preview_items, + ) + except Exception: + pass + for idx, run_args in enumerate(selection_runs, 1): + run_label = self._selection_run_label( + run_args, + extra_url_prefixes=selection_url_prefixes, + ) + try: + progress.set_status( + f"downloading {idx}/{total_selection}: {run_label}" + ) + except Exception: + pass + try: + if isinstance(config, dict): + config["_download_file_batch_total"] = total_selection + config["_download_file_batch_index"] = idx + config["_download_file_batch_label"] = run_label + config["_download_file_suppress_nested_pipe_progress"] = True + except Exception: + pass + exit_code = self._run_impl(None, run_args, config) + if exit_code == 0: + successes += 1 + else: + failures += 1 + last_code = exit_code + + piped_items = residual_items + if not piped_items: + if successes > 0: + return 0 + return last_code or 1 + finally: + try: + if isinstance(config, dict): + if original_skip_preflight is None: + config.pop("_skip_url_preflight", None) + else: + config["_skip_url_preflight"] = original_skip_preflight + if original_timeout is None: + config.pop("_pipeobject_timeout_seconds", None) + else: + config["_pipeobject_timeout_seconds"] = original_timeout + if original_skip_direct is None: + config.pop("_skip_direct_on_streaming_failure", None) + else: + config["_skip_direct_on_streaming_failure"] = original_skip_direct + if original_batch_total is None: + config.pop("_download_file_batch_total", None) + else: + config["_download_file_batch_total"] = original_batch_total + if original_batch_index is None: + config.pop("_download_file_batch_index", None) + else: + config["_download_file_batch_index"] = original_batch_index + if original_batch_label is None: + config.pop("_download_file_batch_label", None) + else: + config["_download_file_batch_label"] = original_batch_label + if original_suppress_nested is None: + config.pop("_download_file_suppress_nested_pipe_progress", None) + else: + config["_download_file_suppress_nested_pipe_progress"] = original_suppress_nested + except Exception: + pass + + had_piped_input = False + try: + if isinstance(result, list): + had_piped_input = bool(result) + else: + had_piped_input = bool(result) + except Exception: + had_piped_input = False + + if (had_piped_input and raw_url and len(raw_url) == 1 + and (not parsed.get("path"))): + candidate = str(raw_url[0] or "").strip() + low = candidate.lower() + looks_like_url = low.startswith( + ("http://", "https://", "ftp://", "magnet:", "torrent:") + + tuple(selection_url_prefixes) + ) + looks_like_provider = ( + ":" in candidate and not candidate.startswith( + ("http:", "https:", "ftp:", "ftps:", "file:") + + tuple(selection_url_prefixes) + ) + ) + looks_like_windows_path = ( + (len(candidate) >= 2 and candidate[1] == ":") + or candidate.startswith("\\\\") or candidate.startswith("\\") + or candidate.endswith(("\\", "/")) + ) + if (not looks_like_url) and ( + not looks_like_provider) and looks_like_windows_path: + parsed["path"] = candidate + raw_url = [] + piped_items = sh.normalize_result_items(result, include_falsey_single=True) + + if not raw_url and not piped_items and not local_source_inputs: + log("No url or piped items to download", file=sys.stderr) + return 1 + + picker_result = self._maybe_show_provider_picker( + raw_urls=raw_url, + piped_items=piped_items, + parsed=parsed, + config=config, + registry=registry, + ) + if picker_result is not None: + return int(picker_result) + + picker_result = self._maybe_show_provider_picker( + raw_urls=raw_url, + piped_items=piped_items, + parsed=parsed, + config=config, + registry=registry, + ) + if picker_result is not None: + return int(picker_result) + + final_output_dir = resolve_target_dir(parsed, config) + if not final_output_dir: + return 1 + + try: + debug_panel( + "download-file plan", + [ + ("output_dir", final_output_dir), + ("remaining_urls", len(raw_url)), + ("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))), + ], + border_style="cyan", + ) + except Exception: + pass + + total_items = max(1, len(raw_url or []) + len(piped_items or [])) + preview = build_pipeline_preview(raw_url, piped_items) + + progress.ensure_local_ui( + label="download-file", + total_items=total_items, + items_preview=preview + ) + + raw_url, preflight_exit, skipped_dupe_count = self._preflight_explicit_url_duplicates( + raw_urls=raw_url, + config=config, + ) + if preflight_exit is not None: + return int(preflight_exit) + + downloaded_count = 0 + + if local_source_inputs: + downloaded_count += self._process_explicit_local_sources( + local_sources=local_source_inputs, + final_output_dir=final_output_dir, + parsed=parsed, + progress=progress, + config=config, + ) + + storage_downloaded, piped_items, storage_exit = self._process_storage_items( + piped_items=piped_items, + parsed=parsed, + config=config, + final_output_dir=final_output_dir, + ) + downloaded_count += int(storage_downloaded) + if storage_exit is not None: + return int(storage_exit) + + if skipped_dupe_count and not raw_url and not piped_items: + return 0 if downloaded_count > 0 else 0 + + urls_downloaded, early_exit = self._process_explicit_urls( + raw_urls=raw_url, + final_output_dir=final_output_dir, + config=config, + quiet_mode=quiet_mode, + registry=registry, + progress=progress, + parsed=parsed, + args=args, + context_items=(result if isinstance(result, list) else ([result] if result else [])), + ) + downloaded_count += int(urls_downloaded) + if early_exit is not None: + return int(early_exit) + + provider_downloaded, magnet_submissions = self._process_provider_items( + piped_items=piped_items, + final_output_dir=final_output_dir, + config=config, + quiet_mode=quiet_mode, + registry=registry, + progress=progress, + ) + downloaded_count += provider_downloaded + + if downloaded_count > 0 or magnet_submissions > 0: + self._maybe_render_download_details(config=config) + return 0 + + return 1 + + except Exception as e: + log(f"Error in download-file: {e}", file=sys.stderr) + return 1 + + finally: + try: + if isinstance(config, dict): + if had_progress_key: + config["_pipeline_progress"] = prev_progress + else: + config.pop("_pipeline_progress", None) + except Exception: + pass + progress.close_local_ui(force_complete=True) + + +# Late-binding: import sub-module functions and attach them as methods. +from .download_fetch import ( # noqa: E402 + _emit_plugin_items, + _consume_plugin_download_result, + _process_explicit_urls, + _expand_provider_items, + _process_provider_items, + _download_provider_items, + _emit_local_file, + _resolve_provider_preflight_items, + _build_provider_playlist_item_selector, + _format_timecode, + _rebase_subtitle_timestamp_text, + _format_clip_range, + _apply_clip_decorations, + _parse_clip_spec_to_ranges, +) + +from .download_storage import ( # noqa: E402 + _iter_storage_export_refs, + _export_store_file, + _process_storage_items, + _process_explicit_local_sources, + _extract_hash_from_search_hit, + _iter_duplicate_tag_values, + _extract_duplicate_namespace_tags, + _extract_duplicate_title_tag, + _extract_duplicate_title, + _has_duplicate_title, + _build_duplicate_display_row, + _fetch_duplicate_metadata_for_hash, + _enrich_duplicate_metadata, + _fetch_duplicate_metadata_for_hashes, + _collect_existing_url_match_refs_for_url, + _find_existing_url_matches_for_url, + _find_existing_hash_for_url, + _find_existing_hashes_for_url, + _init_storage, + _supports_storage_duplicate_lookup, + _filter_supported_urls, + _canonicalize_url_for_storage, + _preflight_url_duplicate, + _preflight_explicit_url_duplicates, + _download_supported_urls, + _maybe_show_playlist_table, + _maybe_show_format_table_for_single_url, + _run_streaming_urls, +) + +Download_File._emit_plugin_items = _emit_plugin_items +Download_File._consume_plugin_download_result = _consume_plugin_download_result +Download_File._process_explicit_urls = _process_explicit_urls +Download_File._expand_provider_items = _expand_provider_items +Download_File._process_provider_items = _process_provider_items +Download_File._download_provider_items = _download_provider_items +Download_File._emit_local_file = _emit_local_file +Download_File._resolve_provider_preflight_items = staticmethod(_resolve_provider_preflight_items) +Download_File._build_provider_playlist_item_selector = staticmethod(_build_provider_playlist_item_selector) +Download_File._format_timecode = staticmethod(_format_timecode) +Download_File._rebase_subtitle_timestamp_text = staticmethod(_rebase_subtitle_timestamp_text) +Download_File._format_clip_range = classmethod(_format_clip_range) +Download_File._apply_clip_decorations = classmethod(_apply_clip_decorations) +Download_File._parse_clip_spec_to_ranges = staticmethod(_parse_clip_spec_to_ranges) + +Download_File._iter_storage_export_refs = staticmethod(_iter_storage_export_refs) +Download_File._export_store_file = _export_store_file +Download_File._process_storage_items = _process_storage_items +Download_File._process_explicit_local_sources = _process_explicit_local_sources +Download_File._extract_hash_from_search_hit = classmethod(_extract_hash_from_search_hit) +Download_File._iter_duplicate_tag_values = staticmethod(_iter_duplicate_tag_values) +Download_File._extract_duplicate_namespace_tags = staticmethod(_extract_duplicate_namespace_tags) +Download_File._extract_duplicate_title_tag = staticmethod(_extract_duplicate_title_tag) +Download_File._extract_duplicate_title = classmethod(_extract_duplicate_title) +Download_File._has_duplicate_title = classmethod(_has_duplicate_title) +Download_File._build_duplicate_display_row = classmethod(_build_duplicate_display_row) +Download_File._fetch_duplicate_metadata_for_hash = classmethod(_fetch_duplicate_metadata_for_hash) +Download_File._enrich_duplicate_metadata = classmethod(_enrich_duplicate_metadata) +Download_File._fetch_duplicate_metadata_for_hashes = classmethod(_fetch_duplicate_metadata_for_hashes) +Download_File._collect_existing_url_match_refs_for_url = classmethod(_collect_existing_url_match_refs_for_url) +Download_File._find_existing_url_matches_for_url = classmethod(_find_existing_url_matches_for_url) +Download_File._find_existing_hash_for_url = classmethod(_find_existing_hash_for_url) +Download_File._find_existing_hashes_for_url = classmethod(_find_existing_hashes_for_url) +Download_File._init_storage = staticmethod(_init_storage) +Download_File._supports_storage_duplicate_lookup = staticmethod(_supports_storage_duplicate_lookup) +Download_File._filter_supported_urls = staticmethod(_filter_supported_urls) +Download_File._canonicalize_url_for_storage = staticmethod(_canonicalize_url_for_storage) +Download_File._preflight_url_duplicate = staticmethod(_preflight_url_duplicate) +Download_File._preflight_explicit_url_duplicates = _preflight_explicit_url_duplicates +Download_File._download_supported_urls = _download_supported_urls +Download_File._maybe_show_playlist_table = _maybe_show_playlist_table +Download_File._maybe_show_format_table_for_single_url = _maybe_show_format_table_for_single_url +Download_File._run_streaming_urls = _run_streaming_urls + +CMDLET = Download_File() diff --git a/cmdlet/file/download_fetch.py b/cmdlet/file/download_fetch.py new file mode 100644 index 0000000..27ef37e --- /dev/null +++ b/cmdlet/file/download_fetch.py @@ -0,0 +1,1033 @@ +"""URL/plugin fetching, streaming, and emission for download-file.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence +from collections.abc import Sequence as SequenceABC +from pathlib import Path +import sys + +from SYS.logger import log, debug_panel +from SYS.pipeline_progress import PipelineProgress +from SYS import pipeline as pipeline_context +from SYS.utils import sha256_file +from API.HTTP import download_direct_file + +from .. import _shared as sh + +from .download_core import Download_File + +coerce_to_path = sh.coerce_to_path +coerce_to_pipe_object = sh.coerce_to_pipe_object +register_url_with_local_library = sh.register_url_with_local_library +get_field = sh.get_field + + +def _emit_plugin_items( + self, + *, + items: Sequence[Any], + config: Dict[str, Any], +) -> int: + emitted = 0 + for item in items: + if not isinstance(item, dict): + continue + pipeline_context.emit(item) + if item.get("url"): + try: + pipe_obj = coerce_to_pipe_object(item) + register_url_with_local_library(pipe_obj, config) + except Exception: + pass + emitted += 1 + return emitted + + +def _consume_plugin_download_result( + self, + *, + result: Any, + config: Dict[str, Any], +) -> tuple[int, Optional[int], bool]: + if result is None: + return 0, None, False + + if isinstance(result, list): + if result and all(isinstance(item, dict) for item in result): + return _emit_plugin_items(self, items=result, config=config), 0, True + return 0, None, False + + if not isinstance(result, dict): + return 0, None, False + + action = str( + result.get("action") + or result.get("plugin_action") + or "" + ).strip().lower() + + if action in {"emit_items", "emit_pipe_objects"}: + items = result.get("items") or [] + exit_code = result.get("exit_code") + emitted = _emit_plugin_items( + self, + items=items if isinstance(items, list) else [], + config=config, + ) + try: + normalized_exit = int(exit_code) if exit_code is not None else 0 + except Exception: + normalized_exit = 0 + return emitted, normalized_exit, True + + if action == "handled": + exit_code = result.get("exit_code") + try: + normalized_exit = int(exit_code) if exit_code is not None else 0 + except Exception: + normalized_exit = 0 + try: + downloaded = int(result.get("downloaded") or 0) + except Exception: + downloaded = 0 + return downloaded, normalized_exit, True + + return 0, None, False + + +def _emit_local_file( + self, + *, + downloaded_path: Path, + source: Optional[str], + title_hint: Optional[str], + tags_hint: Optional[List[str]], + media_kind_hint: Optional[str], + full_metadata: Optional[Dict[str, Any]], + progress: PipelineProgress, + config: Dict[str, Any], + provider_hint: Optional[str] = None, +) -> None: + title_val = (title_hint or downloaded_path.stem or "Unknown").strip() or downloaded_path.stem + hash_value = sha256_file(downloaded_path) + notes: Optional[Dict[str, str]] = None + try: + if isinstance(full_metadata, dict): + _provider_notes = full_metadata.get("_notes") + if isinstance(_provider_notes, dict) and _provider_notes: + notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v} + except Exception: + notes = None + tag: List[str] = [] + if tags_hint: + tag.extend([str(t) for t in tags_hint if t]) + if not any(str(t).lower().startswith("title:") for t in tag): + tag.insert(0, f"title:{title_val}") + + payload: Dict[str, Any] = { + "path": str(downloaded_path), + "hash": hash_value, + "title": title_val, + "action": "cmdlet:download-file", + "download_mode": "file", + "store": "local", + "media_kind": media_kind_hint or "file", + "tag": tag, + } + if provider_hint: + payload["plugin"] = str(provider_hint) + if full_metadata: + payload["metadata"] = full_metadata + if notes: + payload["notes"] = notes + if source and str(source).startswith("http"): + payload["url"] = source + elif source: + payload["source_url"] = source + + pipeline_context.emit(payload) + + +def _expand_provider_items( + self, + *, + piped_items: Sequence[Any], + registry: Dict[str, Any], + config: Dict[str, Any], +) -> List[Any]: + get_provider = registry.get("get_plugin") + expanded_items: List[Any] = [] + + for item in piped_items: + try: + provider_key = self._provider_key_from_item(item) + provider = get_provider(provider_key, config) if provider_key and get_provider else None + + if provider and hasattr(provider, "expand_item") and callable(provider.expand_item): + try: + sub_items = provider.expand_item(item) + if sub_items: + expanded_items.extend(sub_items) + continue + except Exception as e: + debug_panel( + "download-file expand_item failed", + [ + ("plugin", provider_key), + ("error", e), + ], + border_style="yellow", + ) + + expanded_items.append(item) + except Exception: + expanded_items.append(item) + + return expanded_items + + +def _download_provider_items( + self, + *, + provider: Any, + provider_name: str, + search_result: Any, + output_dir: Path, + progress: PipelineProgress, + quiet_mode: bool, + config: Dict[str, Any], +) -> int: + if provider is None or not hasattr(provider, "download_items"): + return 0 + + def _on_emit(path: Path, file_url: str, relpath: str, metadata: Dict[str, Any]) -> None: + title_hint = None + try: + title_hint = metadata.get("name") or relpath + except Exception: + title_hint = relpath + title_hint = title_hint or (Path(path).name if path else "download") + + _emit_local_file( + self, + downloaded_path=path, + source=file_url, + title_hint=title_hint, + tags_hint=None, + media_kind_hint="file", + full_metadata=metadata if isinstance(metadata, dict) else None, + progress=progress, + config=config, + provider_hint=provider_name, + ) + + try: + downloaded_count = provider.download_items( + search_result, + output_dir, + emit=_on_emit, + progress=progress, + quiet_mode=quiet_mode, + path_from_result=coerce_to_path, + config=config, + ) + except TypeError: + downloaded_count = provider.download_items( + search_result, + output_dir, + emit=_on_emit, + progress=progress, + quiet_mode=quiet_mode, + path_from_result=coerce_to_path, + ) + except Exception as exc: + log(f"Provider {provider_name} download_items error: {exc}", file=sys.stderr) + return 0 + + try: + return int(downloaded_count or 0) + except Exception: + return 0 + + +def _process_explicit_urls( + self, + *, + raw_urls: Sequence[str], + final_output_dir: Path, + config: Dict[str, Any], + quiet_mode: bool, + registry: Dict[str, Any], + progress: PipelineProgress, + parsed: Dict[str, Any], + args: Sequence[str], + context_items: Sequence[Any] = (), +) -> tuple[int, Optional[int]]: + downloaded_count = 0 + skipped_duplicate_only = 0 + attempted_download = False + suppress_nested, batch_total, batch_index, batch_label = self._batch_progress_state(config) + total_urls = len(raw_urls or []) + + try: + if total_urls > 1 and not suppress_nested: + progress.begin_pipe(total_items=total_urls, items_preview=list(raw_urls[:5])) + except Exception: + pass + + SearchResult = registry.get("SearchResult") + get_plugin = registry.get("get_plugin") + match_plugin_name_for_url = registry.get("match_plugin_name_for_url") + + for idx, url in enumerate(raw_urls, 1): + try: + try: + display_total = batch_total if batch_total > 0 else total_urls + display_index = batch_index if batch_total > 0 else idx + display_label = batch_label or str(url) + if display_total > 0: + progress.set_status( + f"downloading {display_index}/{display_total}: {display_label}" + ) + except Exception: + pass + + provider_name = None + if match_plugin_name_for_url: + try: + provider_name = match_plugin_name_for_url(str(url)) + except Exception: + pass + + provider = None + if provider_name and get_plugin: + provider = get_plugin(provider_name, config) + + if provider: + try: + handled = False + if hasattr(provider, "handle_url"): + try: + handled, path = provider.handle_url(str(url), output_dir=final_output_dir) + if handled: + extra_meta = None + title_hint = None + tags_hint: Optional[List[str]] = None + media_kind_hint = None + path_value: Optional[Any] = path + + if isinstance(path, dict): + plugin_action = str( + path.get("action") + or path.get("plugin_action") + or "" + ).strip().lower() + if plugin_action == "download_items" or bool(path.get("download_items")): + request_metadata = path.get("metadata") or path.get("full_metadata") or {} + if not isinstance(request_metadata, dict): + request_metadata = {} + magnet_id = path.get("magnet_id") or request_metadata.get("magnet_id") + if magnet_id is not None: + request_metadata.setdefault("magnet_id", magnet_id) + + if SearchResult is None: + continue + + sr = SearchResult( + table=str(provider_name), + title=str(path.get("title") or path.get("name") or f"{provider_name} item"), + path=str(path.get("path") or path.get("url") or url), + full_metadata=request_metadata, + ) + downloaded_extra = _download_provider_items( + self, + provider=provider, + provider_name=str(provider_name), + search_result=sr, + output_dir=final_output_dir, + progress=progress, + quiet_mode=quiet_mode, + config=config, + ) + if downloaded_extra: + downloaded_count += int(downloaded_extra) + continue + + plugin_downloaded, plugin_exit, plugin_handled = _consume_plugin_download_result( + self, + result=path, + config=config, + ) + if plugin_handled: + downloaded_count += plugin_downloaded + if plugin_exit is not None and plugin_downloaded == 0: + return downloaded_count, int(plugin_exit) + if plugin_downloaded: + continue + + path_value = path.get("path") or path.get("file_path") + extra_meta = path.get("metadata") or path.get("full_metadata") + title_hint = path.get("title") or path.get("name") + media_kind_hint = path.get("media_kind") + tags_val = path.get("tags") or path.get("tag") + if isinstance(tags_val, (list, tuple, set)): + tags_hint = [str(t) for t in tags_val if t] + elif isinstance(tags_val, str) and tags_val.strip(): + tags_hint = [str(tags_val).strip()] + + if path_value: + p_val = Path(str(path_value)) + if not title_hint and isinstance(extra_meta, dict): + title_hint = extra_meta.get("title") or extra_meta.get("name") + + _emit_local_file( + self, + downloaded_path=p_val, + source=str(url), + title_hint=str(title_hint) if title_hint else p_val.stem, + tags_hint=tags_hint, + media_kind_hint=str(media_kind_hint) if media_kind_hint else "file", + full_metadata=extra_meta, + progress=progress, + config=config, + provider_hint=provider_name + ) + downloaded_count += 1 + continue + except Exception as e: + debug_panel( + "download-file provider error", + [ + ("plugin", provider_name), + ("url", url), + ("operation", "handle_url"), + ("error", e), + ], + border_style="yellow", + ) + + if not handled and hasattr(provider, "download_url"): + parsed_for_provider = parsed + provider_preflight_items = _resolve_provider_preflight_items( + provider, + url=str(url), + parsed=parsed, + args=args, + ) + if provider_preflight_items: + provider_preflight_urls = [ + str(item.get("url") or "").strip() + for item in provider_preflight_items + if str(item.get("url") or "").strip() + ] + provider_preflight_urls, preflight_exit, provider_skipped = self._preflight_explicit_url_duplicates( + raw_urls=provider_preflight_urls, + config=config, + ) + if preflight_exit is not None: + return downloaded_count, int(preflight_exit) + if provider_skipped: + if not provider_preflight_urls: + skipped_duplicate_only += 1 + continue + selector = _build_provider_playlist_item_selector( + provider_preflight_items, + remaining_urls=provider_preflight_urls, + ) + if selector: + parsed_for_provider = dict(parsed) + parsed_for_provider["item"] = selector + try: + attempted_download = True + res = provider.download_url( + str(url), + final_output_dir, + parsed=parsed_for_provider, + args=list(args), + progress=progress, + quiet_mode=quiet_mode, + context_items=list(context_items or []), + ) + except TypeError: + attempted_download = True + res = provider.download_url(str(url), final_output_dir) + + plugin_downloaded, plugin_exit, plugin_handled = _consume_plugin_download_result( + self, + result=res, + config=config, + ) + if plugin_handled: + downloaded_count += plugin_downloaded + if plugin_exit is not None and plugin_downloaded == 0: + return downloaded_count, int(plugin_exit) + if plugin_downloaded: + continue + + if res: + p_val = None + extra_meta = None + if isinstance(res, (str, Path)): + p_val = Path(res) + elif isinstance(res, tuple) and len(res) > 0: + p_val = Path(res[0]) + if len(res) > 1 and isinstance(res[1], dict): + extra_meta = res[1] + elif isinstance(res, dict): + path_candidate = res.get("path") or res.get("file_path") + if path_candidate: + p_val = Path(path_candidate) + extra_meta = res + + if p_val: + _emit_local_file( + self, + downloaded_path=p_val, + source=str(url), + title_hint=p_val.stem, + tags_hint=None, + media_kind_hint=extra_meta.get("media_kind") if extra_meta else "file", + full_metadata=extra_meta, + provider_hint=provider_name, + progress=progress, + config=config, + ) + downloaded_count += 1 + continue + + except Exception as e: + log(f"Provider {provider_name} error handling {url}: {e}", file=sys.stderr) + pass + + if not handled: + continue + + # Direct Download Fallback + attempted_download = True + result_obj = download_direct_file( + str(url), + final_output_dir, + quiet=quiet_mode, + pipeline_progress=progress, + ) + downloaded_path = self._path_from_download_result(result_obj) + + _emit_local_file( + self, + downloaded_path=downloaded_path, + source=str(url), + title_hint=downloaded_path.stem, + tags_hint=[f"title:{downloaded_path.stem}"], + media_kind_hint="file", + full_metadata=None, + progress=progress, + config=config, + ) + downloaded_count += 1 + + except Exception as e: + log(f"Download failed for {url}: {e}", file=sys.stderr) + + if downloaded_count == 0 and skipped_duplicate_only > 0 and not attempted_download: + return downloaded_count, 0 + return downloaded_count, None + + +def _process_provider_items(self, + *, + piped_items: Sequence[Any], + final_output_dir: Path, + config: Dict[str, Any], + quiet_mode: bool, + registry: Dict[str, Any], + progress: PipelineProgress, + ) -> tuple[int, int]: + downloaded_count = 0 + queued_magnet_submissions = 0 + get_provider = registry.get("get_plugin") + SearchResult = registry.get("SearchResult") + + expanded_items = _expand_provider_items( + self, + piped_items=piped_items, + registry=registry, + config=config + ) + + total_items = len(expanded_items) + processed_items = 0 + + try: + if total_items: + progress.set_percent(0) + except Exception: + pass + + for idx, item in enumerate(expanded_items, 1): + try: + label = "item" + table = get_field(item, "table") + title = get_field(item, "title") + target = get_field(item, "path") or get_field(item, "url") + + media_kind = get_field(item, "media_kind") + tags_val = get_field(item, "tag") + tags_list: Optional[List[str]] + if isinstance(tags_val, (list, set)): + tags_list = sorted([str(t) for t in tags_val if t]) + else: + tags_list = None + + full_metadata = get_field(item, "full_metadata") + if ((not full_metadata) and isinstance(item, dict) + and isinstance(item.get("extra"), dict)): + extra_md = item["extra"].get("full_metadata") + if isinstance(extra_md, dict): + full_metadata = extra_md + + try: + label = title or target + label = str(label or "item").strip() + if total_items: + pct = int(round((processed_items / max(1, total_items)) * 100)) + progress.set_percent(pct) + progress.set_status( + f"downloading {processed_items + 1}/{total_items}: {label}" + ) + except Exception: + pass + + transfer_label = label + + downloaded_path: Optional[Path] = None + attempted_provider_download = False + provider_sr = None + provider_obj = None + provider_key = self._provider_key_from_item(item) + if provider_key and get_provider and SearchResult: + provider_obj = get_provider(provider_key, config) + + if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): + try: + progress.begin_transfer(label=transfer_label, total=None) + except Exception: + pass + + if provider_obj is not None: + attempted_provider_download = True + sr = SearchResult( + table=str(table), + title=str(title or "Unknown"), + path=str(target or ""), + tag=set(tags_list) if tags_list else set(), + media_kind=str(media_kind or "file"), + full_metadata=full_metadata + if isinstance(full_metadata, dict) else {}, + ) + + output_dir = final_output_dir + + downloaded_path = provider_obj.download(sr, output_dir) + provider_sr = sr + + if downloaded_path is None: + try: + downloaded_extra = _download_provider_items( + self, + provider=provider_obj, + provider_name=str(provider_key), + search_result=sr, + output_dir=output_dir, + progress=progress, + quiet_mode=quiet_mode, + config=config, + ) + except Exception: + downloaded_extra = 0 + + if downloaded_extra: + downloaded_count += int(downloaded_extra) + continue + + if (downloaded_path is None and not attempted_provider_download + and isinstance(target, str) and target.startswith("http")): + + suggested_name = str(title).strip() if title is not None else None + result_obj = download_direct_file( + target, + final_output_dir, + quiet=quiet_mode, + suggested_filename=suggested_name, + pipeline_progress=progress, + ) + downloaded_path = coerce_to_path(result_obj) + + if downloaded_path is None: + log( + f"Cannot download item (no provider handler / unsupported target): {title or target}", + file=sys.stderr, + ) + continue + + if provider_sr is not None: + try: + sr_md = getattr(provider_sr, "full_metadata", None) + if isinstance(sr_md, dict) and sr_md: + full_metadata = sr_md + except Exception: + pass + + try: + if isinstance(full_metadata, dict): + t = str(full_metadata.get("title") or "").strip() + if t: + title = t + except Exception: + pass + + try: + sr_tags = getattr(provider_sr, "tag", None) + if isinstance(sr_tags, (set, list)) and sr_tags: + tags_list = sorted([str(t) for t in sr_tags if t]) + except Exception: + pass + + _emit_local_file( + self, + downloaded_path=downloaded_path, + source=str(target) if target else None, + title_hint=str(title) if title else downloaded_path.stem, + tags_hint=tags_list, + media_kind_hint=str(media_kind) if media_kind else None, + full_metadata=full_metadata if isinstance(full_metadata, dict) else None, + progress=progress, + config=config, + provider_hint=provider_key + ) + downloaded_count += 1 + + except Exception as e: + log(f"Download failed: {e}", file=sys.stderr) + finally: + if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): + try: + progress.finish_transfer(label=transfer_label) + except Exception: + pass + processed_items += 1 + try: + pct = int(round((processed_items / max(1, total_items)) * 100)) + progress.set_percent(pct) + if processed_items >= total_items: + progress.clear_status() + except Exception: + pass + + return downloaded_count, queued_magnet_submissions + + +@staticmethod +def _resolve_provider_preflight_items( + provider: Any, + *, + url: str, + parsed: Dict[str, Any], + args: Sequence[str], +) -> List[Dict[str, Any]]: + resolver = getattr(provider, "resolve_preflight_items", None) + if not callable(resolver): + return [] + + try: + items = resolver(url, parsed=parsed, args=list(args)) + except TypeError: + try: + items = resolver(url) + except Exception: + items = None + except Exception: + items = None + + if not isinstance(items, list): + return [] + + normalized: List[Dict[str, Any]] = [] + for idx, item in enumerate(items, 1): + if not isinstance(item, dict): + continue + item_url = str(item.get("url") or "").strip() + if not item_url: + continue + playlist_index = item.get("playlist_index") + try: + playlist_index_value = int(playlist_index) + except Exception: + playlist_index_value = idx + normalized.append( + { + "url": item_url, + "playlist_index": playlist_index_value, + } + ) + + return normalized + + +@staticmethod +def _build_provider_playlist_item_selector( + items: Sequence[Dict[str, Any]], + *, + remaining_urls: Sequence[str], +) -> Optional[str]: + allowed_urls = { + str(url or "").strip() for url in (remaining_urls or []) if str(url or "").strip() + } + if not allowed_urls: + return None + + selectors: List[str] = [] + for idx, item in enumerate(items, 1): + item_url = str(item.get("url") or "").strip() + if not item_url or item_url not in allowed_urls: + continue + playlist_index = item.get("playlist_index") + try: + playlist_index_value = int(playlist_index) + except Exception: + playlist_index_value = idx + if playlist_index_value <= 0: + continue + selectors.append(str(playlist_index_value)) + + if not selectors: + return None + return ",".join(selectors) + + +@staticmethod +def _format_timecode(seconds: int, *, force_hours: bool) -> str: + total = max(0, int(seconds)) + minutes, secs = divmod(total, 60) + hours, minutes = divmod(minutes, 60) + if force_hours: + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + return f"{minutes:02d}:{secs:02d}" + + +@staticmethod +def _rebase_subtitle_timestamp_text(text: str, offset_seconds: int) -> str: + import re as _re + + if not text: + return text + + try: + offset_value = float(offset_seconds) + except Exception: + return text + + if offset_value <= 0: + return text + + timestamp_re = _re.compile(r"(?<!\d)(?P<ts>(?:\d{2}:)?\d{2}:\d{2}(?:[\\.,]\d{1,3})?)(?!\d)") + + def _shift(match) -> str: + original = str(match.group("ts") or "") + if not original: + return original + + frac_sep = "." + frac_digits = 0 + base = original + frac_seconds = 0.0 + if "." in original: + base, frac = original.split(".", 1) + frac_sep = "." + frac_digits = len(frac) + try: + frac_seconds = float(f"0.{frac}") if frac else 0.0 + except Exception: + frac_seconds = 0.0 + elif "," in original: + base, frac = original.split(",", 1) + frac_sep = "," + frac_digits = len(frac) + try: + frac_seconds = float(f"0.{frac}") if frac else 0.0 + except Exception: + frac_seconds = 0.0 + + parts = base.split(":") + if len(parts) == 3: + hours_s, minutes_s, seconds_s = parts + include_hours = True + elif len(parts) == 2: + hours_s = "0" + minutes_s, seconds_s = parts + include_hours = False + else: + return original + + try: + total = ( + (int(hours_s) * 3600) + + (int(minutes_s) * 60) + + int(seconds_s) + + frac_seconds + + offset_value + ) + except Exception: + return original + + total = max(0.0, total) + whole_seconds = int(total) + fraction = total - whole_seconds + hours, remainder = divmod(whole_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + if frac_digits > 0: + scale = 10 ** frac_digits + frac_value = int(round(fraction * scale)) + if frac_value >= scale: + frac_value = 0 + seconds += 1 + if seconds >= 60: + seconds = 0 + minutes += 1 + if minutes >= 60: + minutes = 0 + hours += 1 + frac_text = f"{frac_value:0{frac_digits}d}" + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + return f"{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + try: + return timestamp_re.sub(_shift, str(text)) + except Exception: + return text + + +@classmethod +def _format_clip_range(cls, start_s: int, end_s: int) -> str: + force_hours = bool(start_s >= 3600 or end_s >= 3600) + return f"{_format_timecode(start_s, force_hours=force_hours)}-{_format_timecode(end_s, force_hours=force_hours)}" + + +@classmethod +def _apply_clip_decorations( + cls, pipe_objects: List[Dict[str, Any]], clip_ranges: List[tuple[int, int]], *, source_king_hash: Optional[str] +) -> None: + if not pipe_objects or len(pipe_objects) != len(clip_ranges): + return + + for po, (start_s, end_s) in zip(pipe_objects, clip_ranges): + clip_range = _format_clip_range(start_s, end_s) + clip_tag = f"clip:{clip_range}" + + po["title"] = clip_tag + + tags = po.get("tag") + if not isinstance(tags, list): + tags = [] + + tags = [t for t in tags if not str(t).strip().lower().startswith("title:")] + tags = [t for t in tags if not str(t).strip().lower().startswith("relationship:")] + tags.insert(0, f"title:{clip_tag}") + + if clip_tag not in tags: + tags.append(clip_tag) + + po["tag"] = tags + + notes = po.get("notes") + if isinstance(notes, dict): + sub_text = notes.get("sub") + if isinstance(sub_text, str) and sub_text.strip(): + notes["sub"] = _rebase_subtitle_timestamp_text(sub_text, start_s) + po["notes"] = notes + + if len(pipe_objects) < 2: + return + + hashes: List[str] = [] + for po in pipe_objects: + h_val = sh.normalize_hash(str(po.get("hash") or "")) + hashes.append(h_val or "") + + king_hash = sh.normalize_hash(source_king_hash) if source_king_hash else None + if not king_hash: + king_hash = hashes[0] if hashes and hashes[0] else None + if not king_hash: + return + + alt_hashes: List[str] = [h for h in hashes if h and h != king_hash] + if not alt_hashes: + return + + for po in pipe_objects: + po["relationships"] = {"king": [king_hash], "alt": list(alt_hashes)} + + +@staticmethod +def _parse_clip_spec_to_ranges(clip_spec: Optional[str]) -> Optional[List[tuple[int, int]]]: + """Parse clip spec strings like '2m-2m20s,5m-6m'.""" + text = str(clip_spec or "").strip() + if not text: + return None + + def _parse_time(value: str) -> Optional[int]: + s = str(value or "").strip().lower() + if not s: + return None + try: + if ":" in s: + parts = [int(p) for p in s.split(":")] + if len(parts) == 2: + return (parts[0] * 60) + parts[1] + if len(parts) == 3: + return (parts[0] * 3600) + (parts[1] * 60) + parts[2] + return None + total = 0 + number = "" + units_seen = False + for ch in s: + if ch.isdigit(): + number += ch + continue + if ch in {"h", "m", "s"} and number: + units_seen = True + val = int(number) + if ch == "h": + total += val * 3600 + elif ch == "m": + total += val * 60 + else: + total += val + number = "" + continue + return None + if number: + total += int(number) + if total == 0 and units_seen: + return 0 + return total if total >= 0 else None + except Exception: + return None + + ranges: List[tuple[int, int]] = [] + for chunk in [c.strip() for c in text.split(",") if c.strip()]: + if "-" not in chunk: + return None + left, right = chunk.split("-", 1) + start = _parse_time(left) + end = _parse_time(right) + if start is None or end is None or end < start: + return None + ranges.append((start, end)) + return ranges or None diff --git a/cmdlet/file/download_storage.py b/cmdlet/file/download_storage.py new file mode 100644 index 0000000..38196d4 --- /dev/null +++ b/cmdlet/file/download_storage.py @@ -0,0 +1,1321 @@ +"""Storage operations, duplicate checking, and metadata for download-file.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence +from collections.abc import Mapping, Sequence as SequenceABC +from pathlib import Path +import sys +import shutil +import webbrowser +from urllib.parse import urlparse +from contextlib import AbstractContextManager, nullcontext + +from SYS.logger import log, debug_panel +from SYS.payload_builders import build_file_result_payload, build_table_result_payload +from SYS.result_table import Table, build_display_row +from SYS.rich_display import stderr_console as get_stderr_console +from SYS import pipeline as pipeline_context +from SYS.utils import sha256_file +from rich.prompt import Prompt +from SYS.selection_builder import build_hash_store_selection +from API.HTTP import download_direct_file + +from .. import _shared as sh + +from .download_core import Download_File + +get_field = sh.get_field +resolve_target_dir = sh.resolve_target_dir +coerce_to_path = sh.coerce_to_path + + +@staticmethod +def _iter_duplicate_tag_values(item: Any) -> List[str]: + def _append_tag(out: List[str], value: Any) -> None: + text = "" + if isinstance(value, bytes): + try: + text = value.decode("utf-8", errors="ignore") + except Exception: + text = str(value) + elif isinstance(value, str): + text = value + if not text: + return + cleaned = text.strip() + if cleaned: + out.append(cleaned) + + def _collect_current(container: Any, out: List[str]) -> None: + if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)): + for tag in container: + _append_tag(out, tag) + return + if not isinstance(container, Mapping): + return + current = container.get("0") + if current is None: + current = container.get(0) + if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)): + for tag in current: + _append_tag(out, tag) + + def _collect_service_data(service_data: Any, out: List[str]) -> None: + if not isinstance(service_data, Mapping): + return + for key in ( + "display_tags", + "display_friendly_tags", + "display", + "storage_tags", + "statuses_to_tags", + "tags", + ): + _collect_current(service_data.get(key), out) + + collected: List[str] = [] + for raw_tags in ( + get_field(item, "tags_flat"), + get_field(item, "tags"), + get_field(item, "tag"), + ): + if isinstance(raw_tags, str): + _append_tag(collected, raw_tags) + continue + if isinstance(raw_tags, (list, tuple, set)): + for raw_tag in raw_tags: + _append_tag(collected, raw_tag) + continue + if not isinstance(raw_tags, Mapping): + continue + + statuses_map = raw_tags.get("service_keys_to_statuses_to_tags") + if isinstance(statuses_map, Mapping): + for status_payload in statuses_map.values(): + _collect_current(status_payload, collected) + + names_map = raw_tags.get("service_keys_to_names") + if isinstance(names_map, Mapping): + _ = names_map + + _collect_service_data(raw_tags, collected) + for maybe_service in raw_tags.values(): + _collect_service_data(maybe_service, collected) + + deduped: List[str] = [] + seen: set[str] = set() + for raw_tag in collected: + text = str(raw_tag or "").strip() + key = text.lower() + if not text or key in seen: + continue + seen.add(key) + deduped.append(text) + return deduped + + +@staticmethod +def _extract_duplicate_namespace_tags(item: Any) -> List[str]: + tag_values = _iter_duplicate_tag_values(item) + + namespace_tags: List[str] = [] + seen: set[str] = set() + for raw_tag in tag_values: + text = str(raw_tag or "").strip() + if not text: + continue + lower = text.lower() + if ":" not in text or lower.startswith("title:"): + continue + if lower in seen: + continue + seen.add(lower) + namespace_tags.append(text) + return namespace_tags + + +@staticmethod +def _extract_duplicate_title_tag(item: Any) -> Optional[str]: + for raw_tag in _iter_duplicate_tag_values(item): + tag_text = str(raw_tag or "").strip() + if not tag_text or not tag_text.lower().startswith("title:"): + continue + value = tag_text.split(":", 1)[1].strip() + if value: + return value + return None + + +@classmethod +def _extract_duplicate_title(cls, item: Any) -> str: + for key in ("title", "name", "filename", "target"): + value = get_field(item, key) + text = str(value or "").strip() + if text: + return text + + tag_title = _extract_duplicate_title_tag(item) + if tag_title: + return tag_title + + path_value = str(get_field(item, "path") or "").strip() + if path_value and not path_value.lower().startswith(("http://", "https://", "file://")): + return path_value + + return "(exists)" + + +@classmethod +def _has_duplicate_title(cls, item: Any) -> bool: + return cls._extract_duplicate_title(item) != "(exists)" + + +@classmethod +def _build_duplicate_display_row( + cls, + item: Any, + *, + backend_name: str, + original_url: str, +) -> Dict[str, Any]: + try: + extracted = build_display_row(item, keys=["title", "store", "hash", "ext", "size"]) + except Exception: + extracted = {} + + title = extracted.get("title") or _extract_duplicate_title(item) + store_name = extracted.get("store") or get_field(item, "store") or backend_name + file_hash = extracted.get("hash") or get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "" + ext_text = str(extracted.get("ext") or get_field(item, "ext") or "").strip() + size_raw = extracted.get("size") + if size_raw is None: + size_raw = get_field(item, "size_bytes") + if size_raw is None: + size_raw = get_field(item, "size") + + if not ext_text: + for candidate in (get_field(item, "path"), get_field(item, "title"), get_field(item, "name")): + candidate_text = str(candidate or "").strip() + if not candidate_text: + continue + suffix = Path(candidate_text).suffix.lstrip(".") + if suffix: + ext_text = suffix + break + + title_text = str(title) + tag_text = ", ".join(_extract_duplicate_namespace_tags(item)) + store_text = str(store_name or backend_name) + file_hash_text = str(file_hash or "") + selection_args = None + selection_action = None + selection_url = None + if file_hash_text and store_text and file_hash_text.strip().lower() != "unknown": + selection_args, selection_action = build_hash_store_selection( + file_hash_text, + store_text, + ) + if selection_args and len(selection_args) >= 2: + normalized_hash = str(selection_args[1]).split("hash:", 1)[-1].strip() + if normalized_hash: + file_hash_text = normalized_hash + selection_url = f"hydrus://{store_text}/{normalized_hash}" + + columns: List[tuple[str, Any]] = [("Title", title_text)] + if tag_text: + columns.append(("Tag", tag_text)) + columns.extend( + [ + ("Store", store_text), + ("Size", size_raw), + ("Ext", ext_text), + ("URL", original_url), + ] + ) + + metadata = dict(item) if isinstance(item, dict) else {} + if file_hash_text: + metadata.setdefault("hash", file_hash_text) + if store_text: + metadata.setdefault("store", store_text) + if ext_text: + metadata.setdefault("ext", ext_text) + if size_raw is not None: + metadata.setdefault("size", size_raw) + metadata.setdefault("size_bytes", size_raw) + metadata.setdefault("url", original_url) + if selection_url: + metadata.setdefault("selection_url", selection_url) + + payload = build_table_result_payload( + title=title_text, + columns=columns, + selection_args=selection_args, + selection_action=selection_action, + store=store_text, + hash=file_hash_text, + ext=ext_text, + size=size_raw, + size_bytes=size_raw, + url=original_url, + tags_flat=metadata.get("tags_flat"), + full_metadata=metadata, + ) + if selection_url: + payload["path"] = selection_url + payload["selection_url"] = selection_url + return payload + + +@classmethod +def _extract_hash_from_search_hit(cls, hit: Any) -> Optional[str]: + if not isinstance(hit, dict): + return None + for key in ("hash", "hash_hex", "file_hash", "hydrus_hash"): + v = hit.get(key) + normalized = sh.normalize_hash(str(v) if v is not None else None) + if normalized: + return normalized + return None + + +@classmethod +def _fetch_duplicate_metadata_for_hash( + cls, + backend: Any, + *, + backend_name: str, + file_hash: str, +) -> Dict[str, Any]: + metadata: Optional[Dict[str, Any]] = None + + fetcher = getattr(backend, "fetch_file_metadata", None) + if callable(fetcher): + try: + payload = fetcher(file_hash) + except TypeError: + try: + payload = fetcher(file_hash=file_hash) + except Exception: + payload = None + except Exception: + payload = None + + if isinstance(payload, dict): + meta_list = payload.get("metadata") + if isinstance(meta_list, list) and meta_list and isinstance(meta_list[0], dict): + metadata = dict(meta_list[0]) + else: + metadata = dict(payload) + + metadata = _enrich_duplicate_metadata( + metadata, + backend, + backend_name=backend_name, + file_hash=file_hash, + ) + + metadata.setdefault("hash", file_hash) + metadata.setdefault("store", backend_name) + return metadata + + +@classmethod +def _enrich_duplicate_metadata( + cls, + metadata: Optional[Dict[str, Any]], + backend: Any, + *, + backend_name: str, + file_hash: str, +) -> Dict[str, Any]: + result = dict(metadata) if isinstance(metadata, dict) else {} + + if result: + extractor = getattr(backend, "_extract_title_and_tags", None) + if callable(extractor): + file_id_value = get_field(result, "file_id") or file_hash + try: + extracted_title, extracted_tags = extractor(result, file_id_value) + except Exception: + extracted_title, extracted_tags = None, None + + if not get_field(result, "tags_flat") and isinstance(extracted_tags, SequenceABC) and not isinstance(extracted_tags, (str, bytes, bytearray, Mapping)): + deduped_tags: List[str] = [] + seen_tags: set[str] = set() + for raw_tag in extracted_tags: + tag_text = str(raw_tag or "").strip() + lowered = tag_text.lower() + if not tag_text or lowered in seen_tags: + continue + seen_tags.add(lowered) + deduped_tags.append(tag_text) + if deduped_tags: + result["tags_flat"] = deduped_tags + + title_text = str(extracted_title or "").strip() + generic_title = f"Hydrus File {file_id_value}".strip() + if title_text and title_text != generic_title: + result.setdefault("title", title_text) + + if not result: + getter = getattr(backend, "get_metadata", None) + if callable(getter): + try: + payload = getter(file_hash) + except Exception: + payload = None + if isinstance(payload, dict): + result = dict(payload) + + getter = getattr(backend, "get_metadata", None) + if callable(getter) and not _has_duplicate_title(result): + try: + getter_payload = getter(file_hash) + except Exception: + getter_payload = None + if isinstance(getter_payload, dict): + for key, value in getter_payload.items(): + current = result.get(key) + if current not in (None, "", [], {}, ()): + continue + if value in (None, "", [], {}, ()): + continue + result[key] = value + + return result + + +@classmethod +def _fetch_duplicate_metadata_for_hashes( + cls, + backend: Any, + *, + backend_name: str, + file_hashes: Sequence[str], +) -> Dict[str, Dict[str, Any]]: + normalized_hashes: List[str] = [] + seen_hashes: set[str] = set() + for raw_hash in file_hashes or []: + normalized_hash = sh.normalize_hash(str(raw_hash) if raw_hash is not None else None) + if not normalized_hash or normalized_hash in seen_hashes: + continue + seen_hashes.add(normalized_hash) + normalized_hashes.append(normalized_hash) + + if not normalized_hashes: + return {} + + metadata_by_hash: Dict[str, Dict[str, Any]] = {} + fetcher = getattr(backend, "fetch_files_metadata", None) + if callable(fetcher): + try: + payload = fetcher( + normalized_hashes, + include_service_keys_to_tags=True, + include_file_url=True, + include_duration=True, + include_size=True, + include_mime=True, + ) + except TypeError: + try: + payload = fetcher( + file_hashes=normalized_hashes, + include_service_keys_to_tags=True, + include_file_url=True, + include_duration=True, + include_size=True, + include_mime=True, + ) + except Exception: + payload = None + except Exception: + payload = None + + if isinstance(payload, dict): + meta_list = payload.get("metadata") + if isinstance(meta_list, list): + for entry in meta_list: + if not isinstance(entry, dict): + continue + entry_hash = sh.normalize_hash(str(entry.get("hash") or entry.get("hash_hex") or entry.get("file_hash") or "")) + if not entry_hash: + continue + metadata_by_hash[entry_hash] = _enrich_duplicate_metadata( + dict(entry), + backend, + backend_name=backend_name, + file_hash=entry_hash, + ) + + for normalized_hash in normalized_hashes: + metadata = metadata_by_hash.get(normalized_hash) + if metadata is None: + metadata = _fetch_duplicate_metadata_for_hash( + backend, + backend_name=backend_name, + file_hash=normalized_hash, + ) + metadata.setdefault("hash", normalized_hash) + metadata.setdefault("store", backend_name) + metadata_by_hash[normalized_hash] = metadata + + return metadata_by_hash + + +@staticmethod +def _init_storage(config: Dict[str, Any]) -> tuple[Any, bool]: + """Initialize store registry and determine whether a Hydrus backend is usable.""" + storage = None + try: + from PluginCore.backend_registry import BackendRegistry as _Store + + storage = _Store(config) + except Exception: + storage = None + + hydrus_available = False + try: + from plugins.hydrusnetwork import api as hydrus_api + + hydrus_available = bool(hydrus_api.is_hydrus_available(config)) + except Exception: + hydrus_available = False + + if storage is not None and not hydrus_available: + try: + backend_names = list(storage.list_backends() or []) + except Exception: + backend_names = [] + for backend_name in backend_names: + try: + backend = storage[backend_name] + except Exception: + continue + if str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": + hydrus_available = True + break + + return storage, hydrus_available + + +@staticmethod +def _supports_storage_duplicate_lookup(raw_url: str) -> bool: + text = str(raw_url or "").strip() + if not text: + return False + + try: + parsed = urlparse(text) + except Exception: + parsed = None + + scheme = str(getattr(parsed, "scheme", "") or "").strip().lower() + return scheme in {"http", "https", "ftp", "ftps"} + + +@staticmethod +def _filter_supported_urls(raw_urls: Sequence[str]) -> tuple[List[str], List[str]]: + """Split explicit URLs into supported and unsupported buckets.""" + supported: List[str] = [] + unsupported: List[str] = [] + for raw in raw_urls or []: + text = str(raw or "").strip() + if not text: + continue + low = text.lower() + if low.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:")): + supported.append(text) + else: + unsupported.append(text) + return supported, unsupported + + +@staticmethod +def _canonicalize_url_for_storage( + *, + requested_url: str, + provider_name: Optional[str] = None, + provider_instance: Optional[str] = None, + provider_item: Optional[Any] = None, +) -> str: + """Return the URL key used for duplicate preflight lookups.""" + return str(requested_url or "").strip() + + +@staticmethod +def _preflight_url_duplicate( + *, + canonical_url: str, + storage: Any, + hydrus_available: bool, + final_output_dir: Path, + auto_continue_duplicates: bool = True, + force_prompt_in_pipeline: bool = False, +) -> bool: + """Run duplicate URL preflight against configured storage backends.""" + if not canonical_url or storage is None: + return True + return not sh.check_url_exists_in_storage( + urls=[canonical_url], + storage=storage, + hydrus_available=hydrus_available, + final_output_dir=final_output_dir, + auto_continue_duplicates=auto_continue_duplicates, + force_prompt_in_pipeline=force_prompt_in_pipeline, + ) + + +@classmethod +def _collect_existing_url_match_refs_for_url( + cls, + storage: Any, + canonical_url: str, + *, + hydrus_available: bool, + config: Optional[Dict[str, Any]] = None, +) -> List[Dict[str, Any]]: + if not canonical_url: + return [] + if not _supports_storage_duplicate_lookup(canonical_url): + return [] + + config_dict = config if isinstance(config, dict) else {} + refs: List[Dict[str, Any]] = [] + seen_pairs: set[tuple[str, str]] = set() + seen_backends: set[str] = set() + + def _append_ref(backend_name: str, backend: Any, *, item: Any = None, file_hash_hint: Optional[str] = None, is_exact: bool = False) -> None: + normalized_hash = sh.normalize_hash(str(file_hash_hint) if file_hash_hint is not None else None) + if not normalized_hash: + normalized_hash = _extract_hash_from_search_hit(item) + pair_key = (str(backend_name or "").strip().lower(), str(normalized_hash or "")) + if pair_key in seen_pairs: + return + seen_pairs.add(pair_key) + refs.append( + { + "backend_name": str(backend_name or "").strip(), + "backend": backend, + "hash": normalized_hash, + "item": dict(item) if isinstance(item, dict) else item, + "is_exact": bool(is_exact), + } + ) + + def _iter_backends() -> List[tuple[str, Any]]: + backends: List[tuple[str, Any]] = [] + if storage is not None: + try: + backend_names = list(storage.list_searchable_backends() or []) + except Exception: + backend_names = [] + + for backend_name in backend_names: + try: + backend = storage[backend_name] + except Exception: + continue + name_text = str(backend_name).strip() + if not name_text or name_text.lower() == "temp": + continue + key = name_text.lower() + if key in seen_backends: + continue + seen_backends.add(key) + backends.append((name_text, backend)) + + try: + registry_helpers = Download_File._load_provider_registry() + get_plugin = registry_helpers.get("get_plugin") + hydrus_provider = get_plugin("hydrusnetwork", config_dict) if callable(get_plugin) else None + if hydrus_provider is not None: + for backend_name, backend in hydrus_provider.iter_backends(): + name_text = str(backend_name or "").strip() + if not name_text: + continue + key = name_text.lower() + if key in seen_backends: + continue + seen_backends.add(key) + backends.append((name_text, backend)) + except Exception: + pass + + return backends + + for backend_name, backend in _iter_backends(): + try: + if not hydrus_available and str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": + continue + except Exception: + pass + + found_exact = False + lookup_exact = getattr(backend, "find_hashes_by_url", None) + if callable(lookup_exact): + try: + hashes = lookup_exact(canonical_url) or [] + except Exception: + hashes = [] + if isinstance(hashes, (list, tuple, set)): + for existing_hash in hashes: + normalized_hash = sh.normalize_hash(str(existing_hash) if existing_hash is not None else None) + if not normalized_hash: + continue + found_exact = True + _append_ref( + backend_name, + backend, + file_hash_hint=normalized_hash, + is_exact=True, + ) + if found_exact: + continue + + searcher = getattr(backend, "search", None) + if callable(searcher): + try: + hits = searcher(f"url:{canonical_url}", limit=5, minimal=True) or [] + except Exception: + hits = [] + for hit in hits: + _append_ref(backend_name, backend, item=hit) + + return refs + + +@classmethod +def _find_existing_url_matches_for_url( + cls, + storage: Any, + canonical_url: str, + *, + hydrus_available: bool, + config: Optional[Dict[str, Any]] = None, +) -> List[Dict[str, Any]]: + refs = _collect_existing_url_match_refs_for_url( + storage, + canonical_url, + hydrus_available=hydrus_available, + config=config, + ) + if not refs: + return [] + + matches: List[Dict[str, Any]] = [] + exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} + prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} + + for ref in refs: + if not ref.get("is_exact"): + continue + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if not backend_key or not normalized_hash: + continue + bucket = exact_hashes_by_backend.setdefault( + backend_key, + { + "backend_name": backend_name, + "backend": ref.get("backend"), + "hashes": [], + }, + ) + if normalized_hash not in bucket["hashes"]: + bucket["hashes"].append(normalized_hash) + + for backend_key, bucket in exact_hashes_by_backend.items(): + metadata_map = _fetch_duplicate_metadata_for_hashes( + bucket.get("backend"), + backend_name=str(bucket.get("backend_name") or backend_key), + file_hashes=list(bucket.get("hashes") or []), + ) + for normalized_hash, metadata in metadata_map.items(): + prefetched_metadata[(backend_key, normalized_hash)] = metadata + + for ref in refs: + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if ref.get("is_exact") and normalized_hash: + candidate = prefetched_metadata.get((backend_key, normalized_hash)) + if candidate is None: + candidate = _fetch_duplicate_metadata_for_hash( + ref.get("backend"), + backend_name=backend_name, + file_hash=normalized_hash, + ) + else: + item = ref.get("item") + candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} + + if normalized_hash: + candidate.setdefault("hash", normalized_hash) + candidate.setdefault("store", backend_name) + matches.append( + _build_duplicate_display_row( + candidate, + backend_name=backend_name, + original_url=canonical_url, + ) + ) + + return matches + + +@classmethod +def _find_existing_hash_for_url( + cls, storage: Any, canonical_url: str, *, hydrus_available: bool +) -> Optional[str]: + hashes = _find_existing_hashes_for_url( + storage, + canonical_url, + hydrus_available=hydrus_available, + config={}, + ) + return hashes[0] if hashes else None + + +@classmethod +def _find_existing_hashes_for_url( + cls, + storage: Any, + canonical_url: str, + *, + hydrus_available: bool, + config: Optional[Dict[str, Any]] = None, +) -> List[str]: + refs = _collect_existing_url_match_refs_for_url( + storage, + canonical_url, + hydrus_available=hydrus_available, + config=config, + ) + hashes: List[str] = [] + seen_hashes: set[str] = set() + for ref in refs: + normalized = sh.normalize_hash(str(ref.get("hash") or "")) + if not normalized or normalized in seen_hashes: + continue + seen_hashes.add(normalized) + hashes.append(normalized) + return hashes + + +def _preflight_explicit_url_duplicates( + self, + *, + raw_urls: Sequence[str], + config: Dict[str, Any], +) -> tuple[List[str], Optional[int], int]: + """Return (urls_to_process, early_exit, skipped_count).""" + urls = [str(u or "").strip() for u in (raw_urls or []) if str(u or "").strip()] + if not urls: + return [], None, 0 + + if bool(config.get("_skip_url_preflight")): + return urls, None, 0 + + storage, hydrus_available = _init_storage(config) + duplicate_refs: Dict[str, List[Dict[str, Any]]] = {} + exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} + for url in urls: + refs = _collect_existing_url_match_refs_for_url( + storage, + url, + hydrus_available=hydrus_available, + config=config, + ) + if not refs: + continue + duplicate_refs[url] = refs + for ref in refs: + if not ref.get("is_exact"): + continue + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if not backend_key or not normalized_hash: + continue + bucket = exact_hashes_by_backend.setdefault( + backend_key, + { + "backend_name": backend_name, + "backend": ref.get("backend"), + "hashes": [], + }, + ) + if normalized_hash not in bucket["hashes"]: + bucket["hashes"].append(normalized_hash) + + if not duplicate_refs: + return urls, None, 0 + + prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} + for backend_key, bucket in exact_hashes_by_backend.items(): + metadata_map = _fetch_duplicate_metadata_for_hashes( + bucket.get("backend"), + backend_name=str(bucket.get("backend_name") or backend_key), + file_hashes=list(bucket.get("hashes") or []), + ) + for normalized_hash, metadata in metadata_map.items(): + prefetched_metadata[(backend_key, normalized_hash)] = metadata + + duplicates: Dict[str, List[Dict[str, Any]]] = {} + for url, refs in duplicate_refs.items(): + rows: List[Dict[str, Any]] = [] + for ref in refs: + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if ref.get("is_exact") and normalized_hash: + candidate = prefetched_metadata.get((backend_key, normalized_hash)) + if candidate is None: + candidate = _fetch_duplicate_metadata_for_hash( + ref.get("backend"), + backend_name=backend_name, + file_hash=normalized_hash, + ) + else: + item = ref.get("item") + candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} + + if normalized_hash: + candidate.setdefault("hash", normalized_hash) + candidate.setdefault("store", backend_name) + rows.append( + _build_duplicate_display_row( + candidate, + backend_name=backend_name, + original_url=url, + ) + ) + if rows: + duplicates[url] = rows + + duplicate_count = len(duplicates) + total_count = len(urls) + try: + debug_panel( + "download-file duplicate preflight", + [ + ("total_urls", total_count), + ("duplicate_urls", duplicate_count), + ], + border_style="yellow", + ) + except Exception: + pass + + table = Table(f"Duplicate URLs detected ({duplicate_count}/{total_count})", max_columns=12) + table._interactive(False) + duplicate_rows: List[Dict[str, Any]] = [] + for _url, rows in duplicates.items(): + for row in rows: + payload = dict(row) if isinstance(row, dict) else {} + duplicate_rows.append(payload) + table.add_result(payload) + + try: + pipeline_context.set_last_result_table_overlay(table, duplicate_rows) + except Exception: + pass + + try: + stdin_interactive = bool(sys.stdin and sys.stdin.isatty()) + except Exception: + stdin_interactive = False + + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): + try: + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + except Exception: + cm = nullcontext() + + policy = "skip" + with cm: + console = get_stderr_console() + try: + console.print(table) + except Exception: + pass + setattr(table, "_rendered_by_cmdlet", True) + + if stdin_interactive: + while True: + try: + raw_policy = Prompt.ask( + "Duplicate URLs found. Action? [I]gnore/[S]kip/[C]ancel", + default="skip", + console=console, + ) + except (EOFError, KeyboardInterrupt): + policy = "cancel" + break + + normalized_policy = Download_File._normalize_duplicate_preflight_policy(raw_policy) + if normalized_policy is not None: + policy = normalized_policy + break + + try: + console.print("Please select one of: I, S, C, ignore, skip, cancel") + except Exception: + pass + else: + policy = "skip" + + if policy == "cancel": + try: + pipeline_context.request_pipeline_stop(reason="duplicate-url cancelled", exit_code=0) + except Exception: + pass + return [], 0, 0 + + if policy == "ignore": + return urls, None, 0 + + filtered = [u for u in urls if u not in duplicates] + skipped = len(urls) - len(filtered) + if skipped: + try: + log(f"Skipped {skipped} duplicate URL(s); processing remaining {len(filtered)}.", file=sys.stderr) + except Exception: + pass + return filtered, None, skipped + + +@staticmethod +def _iter_storage_export_refs( + parsed: Dict[str, Any], + piped_items: Sequence[Any], +) -> tuple[List[Dict[str, Any]], List[Any], Optional[int]]: + refs: List[Dict[str, Any]] = [] + residual_items: List[Any] = [] + + query_text = str(parsed.get("query") or "").strip() + query_hash: Optional[str] = None + if query_text: + query_hash = sh.parse_single_hash_query(query_text) + if query_text.lower().startswith("hash") and not query_hash: + log('Error: -query must be of the form hash:<sha256>', file=sys.stderr) + return [], list(piped_items or []), 1 + + explicit_store = str(parsed.get("instance") or "").strip() + if query_hash: + if not explicit_store: + log('Error: No store name provided', file=sys.stderr) + return [], list(piped_items or []), 1 + refs.append( + { + "hash": query_hash, + "store": explicit_store, + "result": None, + } + ) + + for item in piped_items or []: + normalized_hash = sh.normalize_hash( + str(get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "") + ) + store_name = str(parsed.get("instance") or get_field(item, "store") or "").strip() + if normalized_hash and store_name: + refs.append( + { + "hash": normalized_hash, + "store": store_name, + "result": item, + } + ) + else: + residual_items.append(item) + + return refs, residual_items, None + + +def _export_store_file( + self, + *, + file_hash: str, + store_name: str, + result: Any, + parsed: Dict[str, Any], + config: Dict[str, Any], + final_output_dir: Path, +) -> int: + output_path = parsed.get("path") + explicit_output_requested = bool(output_path) + output_name = parsed.get("name") + browser_flag = bool(parsed.get("browser")) + + backend, _store_registry, _exc = sh.get_preferred_store_backend( + config, + store_name, + suppress_debug=True, + ) + if backend is None: + log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr) + return 1 + + metadata = backend.get_metadata(file_hash) + if not metadata: + log(f"Error: File metadata not found for hash {file_hash}", file=sys.stderr) + return 1 + + try: + debug_panel( + "download-file store export", + [ + ("hash", file_hash), + ("instance", store_name), + ("output_path", output_path or "<default>"), + ("output_name", output_name or "<auto>"), + ("browser", browser_flag), + ], + border_style="blue", + ) + except Exception: + pass + + want_url = browser_flag + source_path = backend.get_file(file_hash, url=want_url) + download_url = None + if isinstance(source_path, str): + if source_path.startswith(("http://", "https://")): + download_url = source_path + else: + source_path = Path(source_path) + + if download_url and (browser_flag or not explicit_output_requested): + try: + webbrowser.open(download_url) + except Exception as exc: + log(f"Error opening browser: {exc}", file=sys.stderr) + return 1 + + pipeline_context.emit( + build_file_result_payload( + title=self._resolve_display_title(result, metadata) or "Opened", + hash_value=file_hash, + store=store_name, + url=download_url, + ) + ) + return 0 + + if download_url is None: + if not source_path or not Path(source_path).exists(): + log(f"Error: Backend could not retrieve file for hash {file_hash}", file=sys.stderr) + return 1 + + filename = str(output_name or "").strip() + if not filename: + title = (metadata.get("title") if isinstance(metadata, dict) else None) or self._resolve_display_title(result, metadata) or "export" + filename = self._sanitize_export_filename(str(title)) + + ext = metadata.get("ext") if isinstance(metadata, dict) else None + if ext and not filename.endswith(str(ext)): + ext_text = str(ext) + if not ext_text.startswith("."): + ext_text = "." + ext_text + filename += ext_text + + if download_url: + result_obj = download_direct_file( + download_url, + final_output_dir, + quiet=True, + suggested_filename=filename, + pipeline_progress=config.get("_pipeline_progress") if isinstance(config, dict) else None, + ) + dest_path = Download_File._path_from_download_result(result_obj) + else: + dest_path = self._unique_export_path(final_output_dir / filename) + shutil.copy2(Path(source_path), dest_path) + + pipeline_context.emit( + build_file_result_payload( + title=filename, + hash_value=file_hash, + store=store_name, + path=str(dest_path), + ) + ) + return 0 + + +def _process_storage_items( + self, + *, + piped_items: Sequence[Any], + parsed: Dict[str, Any], + config: Dict[str, Any], + final_output_dir: Path, +) -> tuple[int, List[Any], Optional[int]]: + refs, residual_items, early_exit = _iter_storage_export_refs(parsed, piped_items) + if early_exit is not None: + return 0, list(residual_items), early_exit + if not refs: + return 0, list(residual_items), None + + successes = 0 + for ref in refs: + exit_code = _export_store_file( + self, + file_hash=str(ref.get("hash") or ""), + store_name=str(ref.get("store") or ""), + result=ref.get("result"), + parsed=parsed, + config=config, + final_output_dir=final_output_dir, + ) + if exit_code != 0: + return successes, list(residual_items), exit_code + successes += 1 + + return successes, list(residual_items), None + + +def _process_explicit_local_sources( + self, + *, + local_sources: Sequence[str], + final_output_dir: Path, + parsed: Dict[str, Any], + progress: Any, + config: Dict[str, Any], +) -> int: + from .download_fetch import _emit_local_file as _emit + + explicit_output_requested = bool(parsed.get("path")) + downloaded_count = 0 + for raw_source in local_sources or []: + source_path = Path(str(raw_source or "")).expanduser() + if not source_path.exists() or not source_path.is_file(): + log(f"File not found: {source_path}", file=sys.stderr) + continue + + if explicit_output_requested: + destination = final_output_dir / source_path.name + destination = self._unique_export_path(destination) + shutil.copy2(source_path, destination) + emit_path = destination + else: + emit_path = source_path + + _emit( + self, + downloaded_path=emit_path, + source=str(source_path), + title_hint=emit_path.stem, + tags_hint=None, + media_kind_hint="file", + full_metadata=None, + progress=progress, + config=config, + ) + downloaded_count += 1 + return downloaded_count + + +# Test-compat wrappers +def _download_supported_urls(self, **kwargs: Any) -> int: + """Download pre-validated streaming URLs (wrapper used by tests).""" + urls = list(kwargs.get("supported_url") or []) + storage = kwargs.get("storage") + hydrus_available = bool(kwargs.get("hydrus_available")) + final_output_dir = kwargs.get("final_output_dir") + skip_preflight = bool(kwargs.get("skip_per_url_preflight")) + + if not urls: + return 1 + + for requested_url in urls: + canonical = _canonicalize_url_for_storage(requested_url=requested_url) + if skip_preflight: + continue + ok = _preflight_url_duplicate( + canonical_url=canonical, + storage=storage, + hydrus_available=hydrus_available, + final_output_dir=Path(final_output_dir) if final_output_dir else Path.cwd(), + ) + if not ok: + continue + + return 0 + + +def _maybe_show_playlist_table(self, **kwargs: Any) -> bool: + """Compat hook used by tests; playlist table rendering is handled elsewhere.""" + return False + + +def _maybe_show_format_table_for_single_url(self, **kwargs: Any) -> Optional[int]: + """Compat hook used by tests; format table rendering is handled elsewhere.""" + return None + + +def _run_streaming_urls( + self, + *, + streaming_urls: Sequence[str], + args: Sequence[str], + config: Dict[str, Any], + parsed: Dict[str, Any], +) -> int: + """Compat wrapper for tests that exercise legacy streaming dispatch flow.""" + from plugins.ytdlp import YtDlpTool + + storage, hydrus_available = _init_storage(config) + supported_url, _unsupported = _filter_supported_urls(streaming_urls) + if not supported_url: + return 1 + + final_output_dir = resolve_target_dir(parsed, config) + if final_output_dir is None: + return 1 + + query_text = str(parsed.get("query") or "") + clip_spec = None + for token in [t.strip() for t in query_text.split(",") if t.strip()]: + if token.lower().startswith("clip:"): + clip_spec = token.split(":", 1)[1].strip() + break + clip_ranges = Download_File._parse_clip_spec_to_ranges(clip_spec) + + ytdlp_tool = YtDlpTool(config) if callable(YtDlpTool) else None + playlist_items = parsed.get("item") + + return _download_supported_urls( + self, + supported_url=supported_url, + ytdlp_tool=ytdlp_tool, + args=list(args), + config=config, + final_output_dir=final_output_dir, + mode="audio", + clip_spec=clip_spec, + clip_ranges=clip_ranges, + query_hash_override=None, + embed_chapters=False, + write_sub=False, + quiet_mode=bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False, + playlist_items=playlist_items, + ytdl_format=(ytdlp_tool.default_format("audio") if ytdlp_tool and hasattr(ytdlp_tool, "default_format") else "best"), + skip_per_url_preflight=False, + forced_single_format_id=None, + forced_single_format_for_batch=False, + formats_cache={}, + storage=storage, + hydrus_available=hydrus_available, + download_timeout_seconds=int(config.get("_pipeobject_timeout_seconds") or 300) if isinstance(config, dict) else 300, + ) diff --git a/cmdlet/file/screenshot.py b/cmdlet/file/screenshot.py index c2883dd..00e1580 100644 --- a/cmdlet/file/screenshot.py +++ b/cmdlet/file/screenshot.py @@ -43,29 +43,17 @@ from SYS import pipeline as pipeline_context # Playwright & Screenshot Dependencies # ============================================================================ -from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool +from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool, USER_AGENT try: from SYS.config import resolve_output_dir except ImportError: - try: - _parent_dir = str(Path(__file__).parent.parent) - if _parent_dir not in sys.path: - sys.path.insert(0, _parent_dir) - from SYS.config import resolve_output_dir - except ImportError: - resolve_output_dir = None + resolve_output_dir = None # ============================================================================ # Screenshot Constants & Configuration # ============================================================================ -USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/120.0.0.0 Safari/537.36" -) - DEFAULT_VIEWPORT: dict[str, int] = { "width": 1920, diff --git a/cmdlet/file/search.py b/cmdlet/file/search.py index 416bd15..a63210a 100644 --- a/cmdlet/file/search.py +++ b/cmdlet/file/search.py @@ -2,16 +2,11 @@ from __future__ import annotations -from typing import Any, Dict, Sequence, List, Optional -from collections import deque +from typing import Any, Dict, Sequence, List, Optional, Tuple import uuid from pathlib import Path import re -import json import sys -import html -import time -from urllib.parse import urlparse, parse_qs, unquote, urljoin from SYS.logger import log, debug, debug_panel from SYS.payload_builders import build_file_result_payload, normalize_file_extension @@ -39,34 +34,17 @@ from .._shared import ( ) from SYS import pipeline as ctx -_WHITESPACE_RE = re.compile(r"\s+") -_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE) -_FILETYPE_TOKEN_RE = re.compile( - r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b", - flags=re.IGNORECASE, +# Web search engine functions extracted to search_engines.py +from .search_engines import ( + _WHITESPACE_RE, + query_web_search, + crawl_site_for_extension, + build_web_search_plan, + _normalize_extension, ) -_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE) -_FILETYPE_REMOVE_RE = re.compile( - r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b", - flags=re.IGNORECASE, -) -_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:") -_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE) -_HTML_TAG_RE = re.compile(r"<[^>]+>") -_DDG_RESULT_ANCHOR_RE = re.compile( - r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>', - flags=re.IGNORECASE | re.DOTALL, -) -_GENERIC_ANCHOR_RE = re.compile( - r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', - flags=re.IGNORECASE | re.DOTALL, -) -_BING_RESULT_ANCHOR_RE = re.compile( - r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', - flags=re.IGNORECASE | re.DOTALL, -) -_STORE_FILTER_RE = re.compile(r"\binstance:([^\s,]+)", flags=re.IGNORECASE) -_STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*instance:[^\s,]+", flags=re.IGNORECASE) + +_STORE_FILTER_RE: re.Pattern = re.compile(r"\binstance:([^\s,]+)", flags=re.IGNORECASE) +_STORE_FILTER_REMOVE_RE: re.Pattern = re.compile(r"\s*[,]?\s*instance:[^\s,]+", flags=re.IGNORECASE) class _WorkerLogger: @@ -216,1107 +194,9 @@ class search_file(Cmdlet): ) self.register() - # --- Helper methods ------------------------------------------------- - @staticmethod - def _normalize_host(value: Any) -> str: - """Normalize host names for matching/filtering.""" - host = str(value or "").strip().lower() - if host.startswith("www."): - host = host[4:] - if ":" in host: - host = host.split(":", 1)[0] - return host - - @classmethod - def _extract_site_host(cls, candidate: Any) -> Optional[str]: - """Extract a host/domain from URL-like input.""" - raw = str(candidate or "").strip().strip('"').strip("'") - if not raw: - return None - - if raw.lower().startswith("site:"): - raw = raw.split(":", 1)[1].strip() - - parsed = None - try: - parsed = urlparse(raw) - except Exception: - parsed = None - - if parsed is None or not getattr(parsed, "hostname", None): - try: - parsed = urlparse(f"https://{raw}") - except Exception: - parsed = None - - host = "" - try: - host = str(getattr(parsed, "hostname", "") or "").strip().lower() - except Exception: - host = "" - - host = cls._normalize_host(host) - if not host or "." not in host: - return None - return host - - @staticmethod - def _normalize_space(text: Any) -> str: - return _WHITESPACE_RE.sub(" ", str(text or "")).strip() - - @classmethod - def _build_web_search_plan( - cls, - *, - query: str, - positional_args: List[str], - storage_backend: Optional[str], - store_filter: Optional[str], - hash_query: List[str], - ) -> Optional[Dict[str, Any]]: - """Build web-search plan for URL + ext/filetype query syntax. - - Example input: - search-file "example.com/foo" -query "ext:pdf" - Produces: - site:example.com filetype:pdf - """ - if storage_backend or store_filter or hash_query: - return None - - text = cls._normalize_space(query) - if not text: - return None - - # Avoid hijacking explicit local search DSL (url:, tag:, hash:, etc.). - local_markers = ("url:", "hash:", "tag:", "instance:", "system:") - if any(marker in text.lower() for marker in local_markers): - return None - - site_host: Optional[str] = None - site_from_positional = False - site_token_to_strip = "" - seed_url = "" - - site_match = _SITE_TOKEN_RE.search(text) - if site_match: - site_host = cls._extract_site_host(site_match.group(1)) - seed_url = str(site_match.group(1) or "").strip() - - if not site_host and positional_args: - site_host = cls._extract_site_host(positional_args[0]) - site_from_positional = bool(site_host) - if site_from_positional: - site_token_to_strip = str(positional_args[0] or "").strip() - seed_url = site_token_to_strip - - if not site_host: - for token in text.split(): - candidate = str(token or "").strip().strip(",") - if not candidate: - continue - lower_candidate = candidate.lower() - if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")): - continue - if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith( - ("http://", "https://") - ): - continue - guessed = cls._extract_site_host(candidate) - if guessed: - site_host = guessed - site_token_to_strip = candidate - break - - if not site_host: - return None - - filetype_match = _FILETYPE_TOKEN_RE.search(text) - filetype = cls._normalize_extension(filetype_match.group(1)) if filetype_match else "" - - # Feature gate: trigger this web-search mode when filetype is present - # or user explicitly provided site: syntax. - has_explicit_site = bool(site_match) - if not filetype and not has_explicit_site: - return None - - residual = text - residual = _SITE_REMOVE_RE.sub(" ", residual) - residual = _FILETYPE_REMOVE_RE.sub(" ", residual) - - if site_from_positional and positional_args: - first = str(positional_args[0] or "").strip() - if first: - residual = re.sub(rf"(?:^|\s){re.escape(first)}(?:\s|$)", " ", residual, count=1) - elif site_token_to_strip: - residual = re.sub( - rf"(?:^|\s){re.escape(site_token_to_strip)}(?:\s|$)", - " ", - residual, - count=1, - ) - - residual = cls._normalize_space(residual) - - search_terms: List[str] = [f"site:{site_host}"] - if filetype: - search_terms.append(f"filetype:{filetype}") - if residual: - search_terms.append(residual) - - search_query = " ".join(search_terms).strip() - if not search_query: - return None - - normalized_seed_url = cls._normalize_seed_url(seed_url, site_host) - - return { - "site_host": site_host, - "filetype": filetype, - "search_query": search_query, - "residual": residual, - "seed_url": normalized_seed_url, - } - - @classmethod - def _normalize_seed_url(cls, seed_value: Any, site_host: str) -> str: - """Build a safe crawl starting URL from user input and resolved host.""" - raw = str(seed_value or "").strip().strip("'\"") - if not raw: - raw = str(site_host or "").strip() - - if raw and not raw.startswith(("http://", "https://")): - raw = f"https://{raw}" - - try: - parsed = urlparse(raw) - except Exception: - parsed = urlparse("") - - target = cls._normalize_host(site_host) - host = cls._normalize_host(getattr(parsed, "hostname", "") or "") - if target and host and not (host == target or host.endswith(f".{target}")): - return f"https://{target}/" - - scheme = str(getattr(parsed, "scheme", "") or "https").lower() - if scheme not in {"http", "https"}: - scheme = "https" - - netloc = str(getattr(parsed, "netloc", "") or "").strip() - if not netloc: - netloc = target - path = str(getattr(parsed, "path", "") or "").strip() - if not path: - path = "/" - - return f"{scheme}://{netloc}{path}" - - @staticmethod - def _is_probable_html_path(path_value: str) -> bool: - """Return True when URL path likely points to an HTML page.""" - path = str(path_value or "").strip() - if not path: - return True - suffix = Path(path).suffix.lower() - if not suffix: - return True - return suffix in {".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".shtml", ".xhtml"} - - @classmethod - def _extract_html_links(cls, *, html_text: str, base_url: str) -> List[str]: - """Extract absolute links from an HTML document.""" - links: List[str] = [] - seen: set[str] = set() - - def _add_link(raw_href: Any) -> None: - href = str(raw_href or "").strip() - if not href or href.startswith(("#", "javascript:", "mailto:")): - return - try: - absolute = urljoin(base_url, href) - parsed = urlparse(absolute) - except Exception: - return - if str(getattr(parsed, "scheme", "") or "").lower() not in {"http", "https"}: - return - clean = parsed._replace(fragment="").geturl() - if clean in seen: - return - seen.add(clean) - links.append(clean) - - try: - from lxml import html as lxml_html - - doc = lxml_html.fromstring(html_text or "") - for node in doc.xpath("//a[@href]"): - _add_link(node.get("href")) - except Exception: - href_pattern = re.compile(r'<a[^>]+href=["\']([^"\']+)["\']', flags=re.IGNORECASE) - for match in href_pattern.finditer(html_text or ""): - _add_link(match.group(1)) - - return links - - @classmethod - def _crawl_site_for_extension( - cls, - *, - seed_url: str, - site_host: str, - extension: str, - limit: int, - max_duration_seconds: float = 15.0, - ) -> List[Dict[str, str]]: - """Fallback crawler that discovers in-site file links by extension.""" - from API.requests_client import get_requests_session - - normalized_ext = cls._normalize_extension(extension) - if not normalized_ext: - return [] - - start_url = cls._normalize_seed_url(seed_url, site_host) - if not start_url: - return [] - - session = get_requests_session() - headers = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" - ), - "Accept-Language": "en-US,en;q=0.9", - } - - queue: deque[str] = deque([start_url]) - queued: set[str] = {start_url} - visited_pages: set[str] = set() - seen_files: set[str] = set() - rows: List[Dict[str, str]] = [] - normalized_limit = max(1, min(int(limit or 1), 100)) - max_pages = max(8, min(normalized_limit * 4, 64)) - crawl_deadline = time.monotonic() + max(5.0, float(max_duration_seconds or 0.0)) - - while ( - queue - and len(visited_pages) < max_pages - and len(rows) < normalized_limit - and time.monotonic() < crawl_deadline - ): - page_url = queue.popleft() - queued.discard(page_url) - if page_url in visited_pages: - continue - visited_pages.add(page_url) - - if time.monotonic() >= crawl_deadline: - break - - try: - response = session.get(page_url, timeout=(4, 8), headers=headers) - response.raise_for_status() - except Exception: - continue - - final_url = str(getattr(response, "url", "") or page_url) - try: - parsed_final = urlparse(final_url) - except Exception: - continue - - final_host = cls._normalize_host(getattr(parsed_final, "hostname", "") or "") - if not cls._url_matches_site(final_url, site_host): - continue - - final_path = str(getattr(parsed_final, "path", "") or "") - direct_ext = cls._normalize_extension(Path(final_path).suffix) - if direct_ext == normalized_ext: - file_url = parsed_final._replace(fragment="").geturl() - if file_url not in seen_files: - seen_files.add(file_url) - title = Path(unquote(final_path)).name or file_url - rows.append( - { - "url": file_url, - "title": title, - "snippet": "Discovered via in-site crawl", - } - ) - continue - - content_type = str((response.headers or {}).get("content-type", "") or "").lower() - if "html" not in content_type and "xhtml" not in content_type: - continue - - html_text = str(getattr(response, "text", "") or "") - if not html_text: - continue - if len(html_text) > 2_500_000: - # Avoid parsing extremely large pages during fallback crawl mode. - continue - - discovered_links = cls._extract_html_links(html_text=html_text, base_url=final_url) - for idx, target in enumerate(discovered_links): - if len(rows) >= normalized_limit: - break - if idx >= 300: - break - if time.monotonic() >= crawl_deadline: - break - try: - parsed_target = urlparse(target) - except Exception: - continue - target_host = cls._normalize_host(getattr(parsed_target, "hostname", "") or "") - if not target_host or not (target_host == final_host or target_host.endswith(f".{site_host}")): - if not cls._url_matches_site(target, site_host): - continue - - target_clean = parsed_target._replace(fragment="").geturl() - target_path = str(getattr(parsed_target, "path", "") or "") - target_ext = cls._normalize_extension(Path(target_path).suffix) - - if target_ext == normalized_ext: - if target_clean in seen_files: - continue - seen_files.add(target_clean) - title = Path(unquote(target_path)).name or target_clean - rows.append( - { - "url": target_clean, - "title": title, - "snippet": f"Discovered via crawl from {final_path or '/'}", - } - ) - continue - - if cls._is_probable_html_path(target_path): - if target_clean not in visited_pages and target_clean not in queued: - queue.append(target_clean) - queued.add(target_clean) - - if time.monotonic() >= crawl_deadline: - debug( - "Web crawl fallback reached time budget", - { - "site": site_host, - "visited_pages": len(visited_pages), - "queued_pages": len(queue), - "results": len(rows), - "time_budget_seconds": max_duration_seconds, - }, - ) - - return rows[:normalized_limit] - - @staticmethod - def _extract_duckduckgo_target_url(href: Any) -> str: - """Extract direct target URL from DuckDuckGo result links.""" - raw_href = str(href or "").strip() - if not raw_href: - return "" - - if raw_href.startswith("//"): - raw_href = f"https:{raw_href}" - - if raw_href.startswith("/"): - raw_href = f"https://duckduckgo.com{raw_href}" - - parsed = None - try: - parsed = urlparse(raw_href) - except Exception: - parsed = None - - try: - host = str(getattr(parsed, "hostname", "") or "").strip().lower() - except Exception: - host = "" - - if host.endswith("duckduckgo.com"): - try: - query = parse_qs(str(getattr(parsed, "query", "") or "")) - candidate = (query.get("uddg") or [""])[0] - if candidate: - return str(unquote(candidate)).strip() - except Exception: - pass - - return raw_href - - @staticmethod - def _extract_yahoo_target_url(href: Any) -> str: - """Extract direct target URL from Yahoo redirect links.""" - raw_href = str(href or "").strip() - if not raw_href: - return "" - - # Yahoo result links often look like: - # https://r.search.yahoo.com/.../RU=<url-encoded-target>/RK=... - ru_match = _YAHOO_RU_RE.search(raw_href) - if ru_match: - try: - return str(unquote(ru_match.group(1))).strip() - except Exception: - pass - - # Fallback for query-string variants. - try: - parsed = urlparse(raw_href) - query = parse_qs(str(getattr(parsed, "query", "") or "")) - candidate = (query.get("RU") or query.get("ru") or [""])[0] - if candidate: - return str(unquote(candidate)).strip() - except Exception: - pass - - return raw_href - - @classmethod - def _url_matches_site(cls, url: str, site_host: str) -> bool: - """Return True when URL host is the requested site/subdomain.""" - try: - parsed = urlparse(str(url or "")) - host = cls._normalize_host(getattr(parsed, "hostname", "") or "") - except Exception: - return False - - target = cls._normalize_host(site_host) - if not host or not target: - return False - return host == target or host.endswith(f".{target}") - - @staticmethod - def _itertext_join(node: Any) -> str: - try: - return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()]) - except Exception: - return "" - - @staticmethod - def _html_fragment_to_text(fragment: Any) -> str: - text = _HTML_TAG_RE.sub(" ", str(fragment or "")) - return html.unescape(text) - - @classmethod - def _append_web_result( - cls, - items: List[Dict[str, str]], - seen_urls: set[str], - *, - site_host: str, - url_text: str, - title_text: str, - snippet_text: str, - ) -> None: - url_clean = str(url_text or "").strip() - if not url_clean or not url_clean.startswith(("http://", "https://")): - return - if not cls._url_matches_site(url_clean, site_host): - return - if url_clean in seen_urls: - return - - seen_urls.add(url_clean) - items.append( - { - "url": url_clean, - "title": cls._normalize_space(title_text) or url_clean, - "snippet": cls._normalize_space(snippet_text), - } - ) - - @classmethod - def _parse_web_results_with_fallback( - cls, - *, - html_text: str, - limit: int, - lxml_parser: Any, - regex_parser: Any, - fallback_when_empty: bool = False, - ) -> List[Dict[str, str]]: - """Run an lxml-based parser with an optional regex fallback.""" - items: List[Dict[str, str]] = [] - seen_urls: set[str] = set() - should_run_regex = False - - try: - from lxml import html as lxml_html - - doc = lxml_html.fromstring(html_text or "") - lxml_parser(doc, items, seen_urls) - should_run_regex = fallback_when_empty and not items - except Exception: - should_run_regex = True - - if should_run_regex: - regex_parser(html_text or "", items, seen_urls) - - return items[:limit] - - @classmethod - def _parse_duckduckgo_results( - cls, - *, - html_text: str, - site_host: str, - limit: int, - ) -> List[Dict[str, str]]: - """Parse DuckDuckGo HTML results into normalized rows.""" - def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - result_nodes = doc.xpath("//div[contains(@class, 'result')]") - - for node in result_nodes: - links = node.xpath(".//a[contains(@class, 'result__a')]") - if not links: - continue - - link = links[0] - href = cls._extract_duckduckgo_target_url(link.get("href")) - title = cls._itertext_join(link) - - snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]") - snippet = "" - if snippet_nodes: - snippet = cls._itertext_join(snippet_nodes[0]) - - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text=snippet, - ) - if len(items) >= limit: - break - - def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html): - href = cls._extract_duckduckgo_target_url(match.group(1)) - title_html = match.group(2) - title = cls._html_fragment_to_text(title_html) - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text="", - ) - if len(items) >= limit: - break - - return cls._parse_web_results_with_fallback( - html_text=html_text, - limit=limit, - lxml_parser=_parse_lxml, - regex_parser=_parse_regex, - fallback_when_empty=True, - ) - - @classmethod - def _parse_yahoo_results( - cls, - *, - html_text: str, - site_host: str, - limit: int, - ) -> List[Dict[str, str]]: - """Parse Yahoo HTML search results into normalized rows.""" - def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - for node in doc.xpath("//a[@href]"): - href = cls._extract_yahoo_target_url(node.get("href")) - title = cls._itertext_join(node) - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text="", - ) - if len(items) >= limit: - break - - def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - for match in _GENERIC_ANCHOR_RE.finditer(raw_html): - href = cls._extract_yahoo_target_url(match.group(1)) - title_html = match.group(2) - title = cls._html_fragment_to_text(title_html) - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text="", - ) - if len(items) >= limit: - break - - return cls._parse_web_results_with_fallback( - html_text=html_text, - limit=limit, - lxml_parser=_parse_lxml, - regex_parser=_parse_regex, - ) - - @classmethod - def _query_yahoo( - cls, - *, - search_query: str, - site_host: str, - limit: int, - session: Any, - deadline: Optional[float] = None, - ) -> List[Dict[str, str]]: - """Fetch results from Yahoo search (robust fallback in bot-protected envs).""" - all_rows: List[Dict[str, str]] = [] - seen_urls: set[str] = set() - - max_pages = max(1, min((max(1, int(limit or 1)) + 9) // 10, 3)) - for page_idx in range(max_pages): - if deadline is not None and time.monotonic() >= deadline: - break - - params = { - "p": search_query, - "n": "10", - "b": str((page_idx * 10) + 1), - } - try: - read_timeout = 10.0 - if deadline is not None: - remaining = max(0.0, float(deadline - time.monotonic())) - if remaining <= 0.0: - break - read_timeout = max(3.0, min(10.0, remaining)) - - response = session.get( - "https://search.yahoo.com/search", - params=params, - timeout=(3, read_timeout), - headers={ - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" - ), - "Accept-Language": "en-US,en;q=0.9", - }, - ) - response.raise_for_status() - except Exception: - break - - page_rows = cls._parse_yahoo_results( - html_text=response.text, - site_host=site_host, - limit=max(1, limit - len(all_rows)), - ) - new_rows = 0 - for row in page_rows: - url_value = str(row.get("url") or "").strip() - if not url_value or url_value in seen_urls: - continue - seen_urls.add(url_value) - all_rows.append(row) - new_rows += 1 - if len(all_rows) >= limit: - break - - if len(all_rows) >= limit or new_rows == 0: - break - - return all_rows[:limit] - - @classmethod - def _parse_bing_results( - cls, - *, - html_text: str, - site_host: str, - limit: int, - ) -> List[Dict[str, str]]: - """Parse Bing HTML search results into normalized rows.""" - def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]") - - for node in result_nodes: - links = node.xpath(".//h2/a") - if not links: - continue - link = links[0] - href = str(link.get("href") or "").strip() - title = cls._itertext_join(link) - - snippet = "" - for sel in ( - ".//*[contains(@class,'b_caption')]//p", - ".//*[contains(@class,'b_snippet')]", - ".//p", - ): - snip_nodes = node.xpath(sel) - if snip_nodes: - snippet = cls._itertext_join(snip_nodes[0]) - break - - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text=snippet, - ) - if len(items) >= limit: - break - - def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: - for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html): - href = match.group(1) - title = cls._html_fragment_to_text(match.group(2)) - cls._append_web_result( - items, - seen_urls, - site_host=site_host, - url_text=href, - title_text=title, - snippet_text="", - ) - if len(items) >= limit: - break - - return cls._parse_web_results_with_fallback( - html_text=html_text, - limit=limit, - lxml_parser=_parse_lxml, - regex_parser=_parse_regex, - ) - - @classmethod - def _query_web_search( - cls, - *, - search_query: str, - site_host: str, - limit: int, - ) -> List[Dict[str, str]]: - """Execute web search and return parsed result rows. - - Uses Yahoo first (works in environments where Bing/DDG HTML endpoints - are challenge-gated), then Bing, then DuckDuckGo. - """ - from API.requests_client import get_requests_session - - session = get_requests_session() - normalized_limit = max(1, min(int(limit or 1), 100)) - engine_deadline = time.monotonic() + 12.0 - - # Yahoo often remains parseable where other engines challenge bots. - all_rows = cls._query_yahoo( - search_query=search_query, - site_host=site_host, - limit=normalized_limit, - session=session, - deadline=engine_deadline, - ) - if all_rows: - return all_rows[:normalized_limit] - - # Bing reliably supports filetype: and site: operators when not challenged. - all_rows = cls._query_bing( - search_query=search_query, - site_host=site_host, - limit=normalized_limit, - session=session, - deadline=engine_deadline, - ) - if all_rows: - return all_rows[:normalized_limit] - - # DDG fallback. - all_rows_ddg: List[Dict[str, str]] = [] - seen_urls: set[str] = set() - endpoints = [ - "https://html.duckduckgo.com/html/", - "https://duckduckgo.com/html/", - ] - for endpoint in endpoints: - if time.monotonic() >= engine_deadline: - break - max_offsets = min(3, max(1, (normalized_limit + 29) // 30)) - for page_idx in range(max_offsets): - if time.monotonic() >= engine_deadline: - break - offset = page_idx * 30 - params = {"q": search_query, "s": str(offset)} - remaining = max(0.0, float(engine_deadline - time.monotonic())) - if remaining <= 0.0: - break - read_timeout = max(3.0, min(10.0, remaining)) - response = session.get( - endpoint, - params=params, - timeout=(3, read_timeout), - headers={"Referer": "https://duckduckgo.com/"}, - ) - response.raise_for_status() - page_rows = cls._parse_duckduckgo_results( - html_text=response.text, - site_host=site_host, - limit=max(1, normalized_limit - len(all_rows_ddg)), - ) - new_rows = 0 - for row in page_rows: - url_value = str(row.get("url") or "").strip() - if not url_value or url_value in seen_urls: - continue - seen_urls.add(url_value) - all_rows_ddg.append(row) - new_rows += 1 - if len(all_rows_ddg) >= normalized_limit: - break - if len(all_rows_ddg) >= normalized_limit or new_rows == 0: - break - if all_rows_ddg: - break - - return all_rows_ddg[:normalized_limit] - - @classmethod - def _query_bing( - cls, - *, - search_query: str, - site_host: str, - limit: int, - session: Any, - deadline: Optional[float] = None, - ) -> List[Dict[str, str]]: - """Fetch results from Bing (supports filetype: and site: natively).""" - all_rows: List[Dict[str, str]] = [] - seen_urls: set[str] = set() - - page_start = 1 - pages_checked = 0 - max_pages = max(1, min((max(1, int(limit or 1)) + 49) // 50, 3)) - while len(all_rows) < limit and pages_checked < max_pages: - if deadline is not None and time.monotonic() >= deadline: - break - - params = {"q": search_query, "first": str(page_start), "count": "50"} - try: - read_timeout = 10.0 - if deadline is not None: - remaining = max(0.0, float(deadline - time.monotonic())) - if remaining <= 0.0: - break - read_timeout = max(3.0, min(10.0, remaining)) - - response = session.get( - "https://www.bing.com/search", - params=params, - timeout=(3, read_timeout), - headers={ - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" - ), - "Accept-Language": "en-US,en;q=0.9", - }, - ) - response.raise_for_status() - except Exception: - break - - page_rows = cls._parse_bing_results( - html_text=response.text, - site_host=site_host, - limit=max(1, limit - len(all_rows)), - ) - new_rows = 0 - for row in page_rows: - url_value = str(row.get("url") or "").strip() - if not url_value or url_value in seen_urls: - continue - seen_urls.add(url_value) - all_rows.append(row) - new_rows += 1 - if len(all_rows) >= limit: - break - - if new_rows == 0 or len(all_rows) >= limit: - break - page_start += 50 - pages_checked += 1 - - return all_rows - - def _run_web_search( - self, - *, - web_plan: Dict[str, Any], - limit: int, - args_list: List[str], - refresh_mode: bool, - command_title: str, - ) -> int: - """Execute URL-scoped web search and emit downloadable table rows.""" - site_host = str(web_plan.get("site_host") or "").strip().lower() - search_query = str(web_plan.get("search_query") or "").strip() - requested_type = self._normalize_extension(web_plan.get("filetype") or "") - seed_url = str(web_plan.get("seed_url") or "").strip() - - if not site_host or not search_query: - log("Error: invalid website search request", file=sys.stderr) - return 1 - - worker_id = str(uuid.uuid4()) - try: - insert_worker( - worker_id, - "search-file", - title=f"Web Search: {search_query}", - description=f"Site: {site_host}", - ) - except Exception: - pass - - try: - from SYS.result_table import Table - - rows = self._query_web_search( - search_query=search_query, - site_host=site_host, - limit=limit, - ) - - if not rows and requested_type: - debug( - "Web search returned 0 rows; falling back to in-site crawl", - {"site": site_host, "ext": requested_type, "seed_url": seed_url}, - ) - rows = self._crawl_site_for_extension( - seed_url=seed_url or f"https://{site_host}/", - site_host=site_host, - extension=requested_type, - limit=limit, - max_duration_seconds=10.0, - ) - - table = Table(command_title) - table.set_table("web.search") - table.set_source_command("search-file", list(args_list)) - try: - table.set_table_metadata( - { - "plugin": "web", - "site": site_host, - "query": search_query, - "filetype": requested_type, - } - ) - except Exception: - pass - - if not rows: - log(f"No web results found for query: {search_query}", file=sys.stderr) - if refresh_mode: - try: - ctx.set_last_result_table_overlay(table, []) - except Exception: - pass - try: - append_worker_stdout(worker_id, _summarize_worker_results([])) - update_worker(worker_id, status="completed") - except Exception: - pass - return 0 - - results_list: List[Dict[str, Any]] = [] - for row in rows: - target_url = str(row.get("url") or "").strip() - if not target_url: - continue - - source_title = str(row.get("title") or "").strip() - title = source_title or target_url - snippet = self._normalize_space(row.get("snippet") or "") - if len(snippet) > 120: - snippet = f"{snippet[:117].rstrip()}..." - - detected_ext = requested_type - file_name = "" - if not detected_ext: - try: - parsed_path = Path(urlparse(target_url).path) - file_name = Path(unquote(str(parsed_path))).name - detected_ext = self._normalize_extension(parsed_path.suffix) - except Exception: - detected_ext = "" - else: - try: - file_name = Path(unquote(urlparse(target_url).path)).name - except Exception: - file_name = "" - - # For filetype-based web searches, prefer a concise filename title. - if file_name: - title = file_name - - payload = build_file_result_payload( - title=title, - path=target_url, - url=target_url, - source="web", - store="web", - table="web.search", - ext=detected_ext, - detail=snippet, - tag=[f"site:{site_host}"] + ([f"type:{detected_ext}"] if detected_ext else []), - columns=[ - ("Title", title), - ("Type", detected_ext), - ("URL", target_url), - ], - _selection_args=["-url", target_url], - _selection_action=["download-file", "-url", target_url], - ) - - table.add_result(payload) - results_list.append(payload) - ctx.emit(payload) - - publish_result_table(ctx, table, results_list, overlay=refresh_mode) - - ctx.set_current_stage_table(table) - - try: - append_worker_stdout(worker_id, _summarize_worker_results(results_list)) - update_worker(worker_id, status="completed") - except Exception: - pass - - return 0 - - except Exception as exc: - log(f"Web search failed: {exc}", file=sys.stderr) - try: - update_worker(worker_id, status="error") - except Exception: - pass - return 1 - - @staticmethod - def _normalize_extension(ext_value: Any) -> str: - """Sanitize extension strings to alphanumerics and cap at 5 chars.""" - return normalize_file_extension(ext_value) + # --- Helper methods from search_engines.py are patched below --- + # The run() method uses self._build_web_search_plan(), self._run_web_search(), etc. + # These are assigned via late-binding at module bottom. @staticmethod def _normalize_lookup_target(value: Optional[str]) -> str: @@ -1426,29 +306,223 @@ class search_file(Cmdlet): def _ensure_storage_columns(self, payload: Dict[str, Any]) -> Dict[str, Any]: """Ensure storage results have the necessary fields for result_table display.""" - # Ensure we have title field if "title" not in payload: payload["title"] = ( payload.get("name") or payload.get("target") or payload.get("path") or "Result" ) - # Ensure we have ext field if ("ext" not in payload) or (not str(payload.get("ext") or "").strip()): title = str(payload.get("title", "")) path_obj = Path(title) if path_obj.suffix: - payload["ext"] = self._normalize_extension(path_obj.suffix.lstrip(".")) + payload["ext"] = _normalize_extension(path_obj.suffix.lstrip(".")) else: payload["ext"] = payload.get("ext", "") - # Ensure size_bytes is present for display (already set by search_file()) - # result_table will handle formatting it - - # Store search uses explicit columns so TAG can appear right after TITLE. self._set_storage_display_columns(payload) return payload + def _run_multi_plugin_search( + self, + *, + plugin_names: List[str], + original_plugin_arg: str, + instance_name: Optional[str], + query: str, + limit: int, + limit_set: bool, + open_id: Optional[int], + args_list: List[str], + refresh_mode: bool, + config: Dict[str, Any], + ) -> int: + if not plugin_names or not query: + from SYS import pipeline as ctx_mod + progress = None + if hasattr(ctx_mod, "get_pipeline_state"): + progress = ctx_mod.get_pipeline_state().live_progress + if progress: + try: + progress.stop() + except Exception: + pass + log("Error: search-file -plugin requires both plugin and query", file=sys.stderr) + log(f"Usage: {self.usage}", file=sys.stderr) + return 1 + + if not limit_set: + limit = 50 + + from SYS import pipeline as ctx_mod + progress = None + if hasattr(ctx_mod, "get_pipeline_state"): + progress = ctx_mod.get_pipeline_state().live_progress + + providers_map = list_plugins_for_cmdlet("search-file", config) + valid_plugins: List[Tuple[str, Any]] = [] + failed_names: List[str] = [] + + for pname in plugin_names: + provider = get_plugin_for_cmdlet(pname, "search-file", config) + resolved = str(getattr(provider, "name", "") or pname).strip().lower() + if not provider or not providers_map.get(resolved, False): + failed_names.append(pname) + continue + valid_plugins.append((pname, provider)) + + if not valid_plugins: + if progress: + try: + progress.stop() + except Exception: + pass + show_plugin_config_panel(failed_names or plugin_names) + available = [n for n, a in providers_map.items() if a] + if available: + show_available_plugins_panel(available) + return 1 + + if failed_names: + log( + f"Warning: Skipping unconfigured plugins: {', '.join(failed_names)}", + file=sys.stderr, + ) + + worker_id = str(uuid.uuid4()) + try: + insert_worker( + worker_id, + "search-file", + title=f"Search: {query}", + description=f"Plugins: {', '.join(plugin_names)}, Query: {query}", + ) + except Exception: + pass + + try: + results_list: List[Dict[str, Any]] = [] + from SYS.result_table import Table + + table_title = f"Combined Search: {query}" + table = Table(table_title)._perseverance(True) + table_type = "search" + table.set_table(table_type) + table.set_source_command("search-file", args_list) + + total_results = 0 + errors: List[str] = [] + + for pname, provider in valid_plugins: + try: + normalized_query = str(query or "").strip() + provider_filters: Dict[str, Any] = {} + try: + normalized_query, provider_filters = provider.extract_query_arguments(query) + except Exception: + provider_filters = {} + + normalized_query = (normalized_query or "").strip() + effective_query = normalized_query or "*" + search_filters = dict(provider_filters or {}) + if instance_name and not search_filters.get("instance"): + search_filters["instance"] = str(instance_name).strip() + + debug_panel( + f"search-file provider request ({pname})", + [ + ("provider", pname), + ("instance", search_filters.get("instance") or "<default>"), + ("query", effective_query), + ("limit", limit), + ("filters", search_filters or "<none>"), + ], + border_style="cyan", + ) + + results = provider.search( + effective_query, limit=limit, filters=search_filters or None + ) + + debug_panel( + f"search-file provider response ({pname})", + [ + ("provider", pname), + ("results", len(results or [])), + ], + border_style="cyan", + ) + + if not results: + continue + + try: + post = getattr(provider, "postprocess_search_results", None) + if callable(post) and isinstance(results, list): + results, _to, _tmo = post( + query=effective_query, + results=results, + filters=search_filters or None, + limit=int(limit or 0), + table_type="search", + table_meta=None, + ) + except Exception: + pass + + for search_result in results: + item_dict = ( + search_result.to_dict() + if hasattr(search_result, "to_dict") + else dict(search_result) + if isinstance(search_result, dict) + else {"title": str(search_result)} + ) + + if "table" not in item_dict: + item_dict["table"] = table_type + if "source" not in item_dict: + item_dict["source"] = pname + + table.add_result(search_result) + results_list.append(item_dict) + ctx.emit(item_dict) + total_results += 1 + + except Exception as exc: + log(f"Error searching plugin '{pname}': {exc}", file=sys.stderr) + errors.append(pname) + + if not results_list: + log(f"No results found for query: {query}", file=sys.stderr) + try: + append_worker_stdout(worker_id, _summarize_worker_results([])) + update_worker(worker_id, status="completed") + except Exception: + pass + return 0 + + publish_result_table(ctx, table, results_list, overlay=refresh_mode) + ctx.set_current_stage_table(table) + + try: + append_worker_stdout(worker_id, _summarize_worker_results(results_list)) + update_worker(worker_id, status="completed") + except Exception: + pass + + return 0 + + except Exception as exc: + log(f"Error during multi-plugin search: {exc}", file=sys.stderr) + import traceback + debug(traceback.format_exc()) + try: + update_worker(worker_id, status="error") + except Exception: + pass + return 1 + def _run_plugin_search( self, *, @@ -1477,20 +551,19 @@ class search_file(Cmdlet): log("Error: search-file -plugin requires both plugin and query", file=sys.stderr) log(f"Usage: {self.usage}", file=sys.stderr) - + providers_map = list_plugins_for_cmdlet("search-file", config) available = [n for n, a in providers_map.items() if a] unconfigured = [n for n, a in providers_map.items() if not a] - + if unconfigured: show_plugin_config_panel(unconfigured) - + if available: show_available_plugins_panel(available) - + return 1 - # Align with plugin default when user did not set -limit. if not limit_set: limit = 50 @@ -1498,7 +571,7 @@ class search_file(Cmdlet): progress = None if hasattr(ctx_mod, "get_pipeline_state"): progress = ctx_mod.get_pipeline_state().live_progress - + providers_map = list_plugins_for_cmdlet("search-file", config) provider = get_plugin_for_cmdlet(plugin_name, "search-file", config) resolved_plugin_name = str(getattr(provider, "name", "") or plugin_name).strip().lower() @@ -1510,7 +583,7 @@ class search_file(Cmdlet): pass show_plugin_config_panel([plugin_name]) - + available = [n for n, a in providers_map.items() if a] if available: show_available_plugins_panel(available) @@ -1535,21 +608,19 @@ class search_file(Cmdlet): provider_text = str(plugin_name or "").strip() provider_lower = provider_text.lower() - # Dynamic query/filter extraction via provider normalized_query = str(query or "").strip() provider_filters: Dict[str, Any] = {} try: normalized_query, provider_filters = provider.extract_query_arguments(query) except Exception: provider_filters = {} - + normalized_query = (normalized_query or "").strip() query = normalized_query or "*" search_filters = dict(provider_filters or {}) if instance_name and not search_filters.get("instance"): search_filters["instance"] = str(instance_name).strip() - # Dynamic table generation via provider table_title = provider.get_table_title(query, search_filters).strip().rstrip(":") table_type = provider.get_table_type(query, search_filters) table_meta = provider.get_table_metadata(query, search_filters) @@ -1561,8 +632,7 @@ class search_file(Cmdlet): table.set_table_metadata(table_meta) except Exception: pass - - # Dynamic source command via provider + source_cmd, source_args = provider.get_source_command(args_list) table.set_source_command(source_cmd, source_args) @@ -1588,7 +658,6 @@ class search_file(Cmdlet): border_style="cyan", ) - # Allow plugins to apply plugin-specific UX transforms (e.g. auto-expansion) try: post = getattr(provider, "postprocess_search_results", None) if callable(post) and isinstance(results, list): @@ -1633,11 +702,9 @@ class search_file(Cmdlet): if "table" not in item_dict: item_dict["table"] = table_type - # Ensure plugin source is present so downstream cmdlets can resolve the owner. if "source" not in item_dict: item_dict["source"] = plugin_name - row_index = len(table.rows) table.add_result(search_result) results_list.append(item_dict) @@ -1669,23 +736,23 @@ class search_file(Cmdlet): # --- Execution ------------------------------------------------------ def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: """Search storage backends for files by various criteria. - + Supports searching by: - Hash (-query "hash:...") - Title (-query "title:...") - Tag (-query "tag:...") - URL (-query "url:...") - Other backend-specific fields - + Optimizations: - Extracts tags from metadata response (avoids duplicate API calls) - Only calls get_tag() separately for backends that don't include tags - + Args: result: Piped input (typically empty for new search) args: Search criteria and options config: Application configuration - + Returns: 0 on success, 1 on error """ @@ -1723,18 +790,15 @@ class search_file(Cmdlet): try: raw_title = ( ctx.get_current_stage_text("") - if hasattr(ctx, - "get_current_stage_text") else None + if hasattr(ctx, "get_current_stage_text") else None ) except Exception: raw_title = None command_title = (str(raw_title).strip() if raw_title else "") or _format_command_title("search-file", - list(args_list)) + list(args_list)) - # Build dynamic flag variants from cmdlet arg definitions. - # This avoids hardcoding flag spellings in parsing loops. flag_registry = self.build_flag_registry() query_flags = { f.lower() @@ -1764,7 +828,6 @@ class search_file(Cmdlet): | open_flags ) - # Parse arguments query = "" storage_backend: Optional[str] = None instance_name: Optional[str] = None @@ -1834,6 +897,8 @@ class search_file(Cmdlet): query = f"{query} {arg}".strip() if query else arg i += 1 else: + if arg.startswith("-"): + log(f"Warning: unrecognized flag '{arg}'", file=sys.stderr) i += 1 query = query.strip() @@ -1844,8 +909,22 @@ class search_file(Cmdlet): if plugin_name: if storage_backend and not instance_name: instance_name = storage_backend + plugin_names = [p.strip() for p in re.split(r"[\s,]+", plugin_name) if p.strip()] + if len(plugin_names) > 1: + return self._run_multi_plugin_search( + plugin_names=plugin_names, + original_plugin_arg=plugin_name, + instance_name=instance_name, + query=query, + limit=limit, + limit_set=limit_set, + open_id=open_id, + args_list=args_list, + refresh_mode=refresh_mode, + config=config, + ) return self._run_plugin_search( - plugin_name=plugin_name, + plugin_name=plugin_names[0] if plugin_names else plugin_name, instance_name=instance_name, query=query, limit=limit, @@ -1891,12 +970,11 @@ class search_file(Cmdlet): return 1 worker_id = str(uuid.uuid4()) - + from PluginCore.backend_registry import BackendRegistry storage_registry = BackendRegistry(config=config or {}) if not storage_registry.list_backends(): - # Internal refreshes should not trigger config panels or stop progress. if "-internal-refresh" in args_list: return 1 @@ -1912,7 +990,6 @@ class search_file(Cmdlet): show_store_config_panel(["Hydrus Network"]) return 1 - # Use a lightweight worker logger to track search results in the central DB with _WorkerLogger(worker_id) as db: try: if "-internal-refresh" not in args_list: @@ -1943,9 +1020,7 @@ class search_file(Cmdlet): backend_to_search = storage_backend or None - if hash_query: - # Explicit hash list search: build rows from backend metadata. backends_to_try: List[str] = [] if backend_to_search: backends_to_try = [backend_to_search] @@ -1968,7 +1043,6 @@ class search_file(Cmdlet): if backend is None: continue try: - # If get_metadata works, consider it a hit; get_file can be optional (e.g. remote URL). meta = backend.get_metadata(h) if meta is None: continue @@ -1984,22 +1058,16 @@ class search_file(Cmdlet): found_any = True searched_backends.append(resolved_backend_name) - # Resolve a path/URL string if possible path_str: Optional[str] = None - # Avoid calling get_file() for remote backends during search/refresh. - meta_obj: Dict[str, - Any] = {} + meta_obj: Dict[str, Any] = {} try: meta_obj = resolved_backend.get_metadata(h) or {} except Exception: meta_obj = {} - # Extract tags from metadata response instead of separate get_tag() call - # Metadata already includes tags if fetched with include_service_keys_to_tags=True tags_list: List[str] = [] - - # First try to extract from metadata tags dict + metadata_tags = meta_obj.get("tags") if isinstance(metadata_tags, dict): collected_tags: List[str] = [] @@ -2031,9 +1099,7 @@ class search_file(Cmdlet): seen_tags.add(key) dedup.append(tag_text) tags_list = dedup - - # Fallback: if metadata didn't include tags, call get_tag() separately - # (This maintains compatibility with backends that don't include tags in metadata) + if not tags_list: try: tag_result = resolved_backend.get_tag(h) @@ -2099,8 +1165,6 @@ class search_file(Cmdlet): if found_any: table.title = command_title - # Add-file refresh quality-of-life: if exactly 1 item is being refreshed, - # show the detailed item panel instead of a single-row table. if refresh_mode and len(results_list) == 1: try: from SYS.rich_display import render_item_details_panel @@ -2175,8 +1239,6 @@ class search_file(Cmdlet): suppress_debug=True, ) if backend is None: - # Configured backend name exists but has no registered implementation or failed to load. - # (e.g. 'all-debrid' being treated as a store but having no store provider). continue searched_backends.append(backend_name) @@ -2205,9 +1267,7 @@ class search_file(Cmdlet): def _as_dict(obj: Any) -> Dict[str, Any]: if isinstance(obj, dict): return dict(obj) - if hasattr(obj, - "to_dict") and callable(getattr(obj, - "to_dict")): + if hasattr(obj, "to_dict") and callable(getattr(obj, "to_dict")): return obj.to_dict() # type: ignore[arg-type] return { "title": str(obj) @@ -2218,14 +1278,12 @@ class search_file(Cmdlet): store_val = str(item_dict.get("store") or "").lower() if store_filter != store_val: continue - - # Normalize storage results (ensure title, ext, etc.) + normalized = self._ensure_storage_columns(item_dict) - # If normalize skipped it due to STORAGE_ORIGINS, do it manually if "title" not in normalized: normalized["title"] = ( - item_dict.get("title") or item_dict.get("name") or + item_dict.get("title") or item_dict.get("name") or item_dict.get("path") or item_dict.get("target") or "Result" ) if "ext" not in normalized: @@ -2233,7 +1291,6 @@ class search_file(Cmdlet): if "." in t: normalized["ext"] = t.split(".")[-1].lower()[:5] - # Make hash/store available for downstream cmdlet without rerunning search hash_val = normalized.get("hash") store_val = normalized.get("store") or item_dict.get("store") or backend_to_search if hash_val and not normalized.get("hash"): @@ -2241,7 +1298,6 @@ class search_file(Cmdlet): if store_val and not normalized.get("store"): normalized["store"] = store_val - # Populate default selection args for interactive @N selection/hash/url handling try: sel_args, sel_action = build_default_selection( path_value=normalized.get("path") or normalized.get("target") or normalized.get("url"), @@ -2262,7 +1318,6 @@ class search_file(Cmdlet): table.title = command_title - # If exactly 1 item is being refreshed, show the detailed item panel. if refresh_mode and len(results_list) == 1: try: from SYS.rich_display import render_item_details_panel @@ -2272,14 +1327,12 @@ class search_file(Cmdlet): pass if refresh_mode: - # For internal refresh, use overlay mode to avoid adding to history try: - # Parse out the store/hash context if possible subject_context = None if "hash:" in query: subject_hash = query.split("hash:")[1].split(",")[0].strip() subject_context = {"store": backend_to_search, "hash": subject_hash} - + publish_result_table( ctx, table, @@ -2319,5 +1372,207 @@ class search_file(Cmdlet): pass return 1 + def _run_web_search( + self, + *, + web_plan: Dict[str, Any], + limit: int, + args_list: List[str], + refresh_mode: bool, + command_title: str, + ) -> int: + """Execute URL-scoped web search and emit downloadable table rows.""" + site_host = str(web_plan.get("site_host") or "").strip().lower() + search_query = str(web_plan.get("search_query") or "").strip() + requested_type = _normalize_extension(web_plan.get("filetype") or "") + seed_url = str(web_plan.get("seed_url") or "").strip() + + if not site_host or not search_query: + log("Error: invalid website search request", file=sys.stderr) + return 1 + + worker_id = str(uuid.uuid4()) + try: + insert_worker( + worker_id, + "search-file", + title=f"Web Search: {search_query}", + description=f"Site: {site_host}", + ) + except Exception: + pass + + try: + from SYS.result_table import Table + from pathlib import PurePosixPath as _PP + + rows = query_web_search( + search_query=search_query, + site_host=site_host, + limit=limit, + ) + + if not rows and requested_type: + debug( + "Web search returned 0 rows; falling back to in-site crawl", + {"site": site_host, "ext": requested_type, "seed_url": seed_url}, + ) + rows = crawl_site_for_extension( + seed_url=seed_url or f"https://{site_host}/", + site_host=site_host, + extension=requested_type, + limit=limit, + max_duration_seconds=10.0, + ) + + table = Table(command_title) + table.set_table("web.search") + table.set_source_command("search-file", list(args_list)) + try: + table.set_table_metadata( + { + "plugin": "web", + "site": site_host, + "query": search_query, + "filetype": requested_type, + } + ) + except Exception: + pass + + if not rows: + log(f"No web results found for query: {search_query}", file=sys.stderr) + if refresh_mode: + try: + ctx.set_last_result_table_overlay(table, []) + except Exception: + pass + try: + append_worker_stdout(worker_id, _summarize_worker_results([])) + update_worker(worker_id, status="completed") + except Exception: + pass + return 0 + + results_list: List[Dict[str, Any]] = [] + for row in rows: + target_url = str(row.get("url") or "").strip() + if not target_url: + continue + + source_title = str(row.get("title") or "").strip() + title = source_title or target_url + snippet = _WHITESPACE_RE.sub(" ", str(row.get("snippet") or "")).strip() + if len(snippet) > 120: + snippet = f"{snippet[:117].rstrip()}..." + + detected_ext = requested_type + file_name = "" + if not detected_ext: + try: + from urllib.parse import urlparse as _urlparse, unquote as _unquote + parsed_path = _PP(_urlparse(target_url).path) + file_name = _PP(_unquote(str(parsed_path))).name + detected_ext = _normalize_extension(parsed_path.suffix) + except Exception: + detected_ext = "" + else: + try: + from urllib.parse import urlparse as _urlparse, unquote as _unquote + file_name = _PP(_unquote(_urlparse(target_url).path)).name + except Exception: + file_name = "" + + if file_name: + title = file_name + + payload = build_file_result_payload( + title=title, + path=target_url, + url=target_url, + source="web", + store="web", + table="web.search", + ext=detected_ext, + detail=snippet, + tag=[f"site:{site_host}"] + ([f"type:{detected_ext}"] if detected_ext else []), + columns=[ + ("Title", title), + ("Type", detected_ext), + ("URL", target_url), + ], + _selection_args=["-url", target_url], + _selection_action=["download-file", "-url", target_url], + ) + + table.add_result(payload) + results_list.append(payload) + ctx.emit(payload) + + publish_result_table(ctx, table, results_list, overlay=refresh_mode) + + ctx.set_current_stage_table(table) + + try: + append_worker_stdout(worker_id, _summarize_worker_results(results_list)) + update_worker(worker_id, status="completed") + except Exception: + pass + + return 0 + + except Exception as exc: + log(f"Web search failed: {exc}", file=sys.stderr) + try: + update_worker(worker_id, status="error") + except Exception: + pass + return 1 + + +# Late-binding: attach search engine functions as classmethod/staticmethod +from .search_engines import ( # noqa: E402 + _normalize_host, + _extract_site_host, + _normalize_space, + _normalize_seed_url, + _is_probable_html_path, + _extract_html_links, + _extract_duckduckgo_target_url, + _extract_yahoo_target_url, + _url_matches_site, + _itertext_join, + _html_fragment_to_text, + _append_web_result, + _parse_web_results_with_fallback, + parse_duckduckgo_results as _parse_duckduckgo_results, + parse_yahoo_results as _parse_yahoo_results, + query_yahoo as _query_yahoo, + parse_bing_results as _parse_bing_results, + query_bing as _query_bing, +) + +search_file._normalize_host = staticmethod(_normalize_host) +search_file._extract_site_host = classmethod(_extract_site_host) +search_file._normalize_space = staticmethod(_normalize_space) +search_file._build_web_search_plan = classmethod(build_web_search_plan) +search_file._normalize_seed_url = classmethod(_normalize_seed_url) +search_file._is_probable_html_path = staticmethod(_is_probable_html_path) +search_file._extract_html_links = classmethod(_extract_html_links) +search_file._crawl_site_for_extension = classmethod(crawl_site_for_extension) +search_file._extract_duckduckgo_target_url = staticmethod(_extract_duckduckgo_target_url) +search_file._extract_yahoo_target_url = staticmethod(_extract_yahoo_target_url) +search_file._url_matches_site = classmethod(_url_matches_site) +search_file._itertext_join = staticmethod(_itertext_join) +search_file._html_fragment_to_text = staticmethod(_html_fragment_to_text) +search_file._append_web_result = classmethod(_append_web_result) +search_file._parse_web_results_with_fallback = classmethod(_parse_web_results_with_fallback) +search_file._parse_duckduckgo_results = classmethod(_parse_duckduckgo_results) +search_file._parse_yahoo_results = classmethod(_parse_yahoo_results) +search_file._query_yahoo = classmethod(_query_yahoo) +search_file._parse_bing_results = classmethod(_parse_bing_results) +search_file._query_web_search = classmethod(query_web_search) +search_file._query_bing = classmethod(_query_bing) + CMDLET = search_file() diff --git a/cmdlet/file/search_engines.py b/cmdlet/file/search_engines.py new file mode 100644 index 0000000..0a2f1e1 --- /dev/null +++ b/cmdlet/file/search_engines.py @@ -0,0 +1,994 @@ +"""Search engine result parsing — Bing, DuckDuckGo, Yahoo, and site crawling. + +Extracted from search.py to keep the cmdlet class focused on orchestration. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence +from collections import deque +from pathlib import Path +import re +import time +import html as _html +from urllib.parse import urlparse, parse_qs, unquote, urljoin + +from SYS.logger import debug +from SYS.payload_builders import normalize_file_extension + +_WHITESPACE_RE = re.compile(r"\s+") +_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE) +_FILETYPE_TOKEN_RE = re.compile( + r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b", + flags=re.IGNORECASE, +) +_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE) +_FILETYPE_REMOVE_RE = re.compile( + r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b", + flags=re.IGNORECASE, +) +_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:") +_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE) +_HTML_TAG_RE = re.compile(r"<[^>]+>") +_DDG_RESULT_ANCHOR_RE = re.compile( + r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>', + flags=re.IGNORECASE | re.DOTALL, +) +_GENERIC_ANCHOR_RE = re.compile( + r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', + flags=re.IGNORECASE | re.DOTALL, +) +_BING_RESULT_ANCHOR_RE = re.compile( + r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', + flags=re.IGNORECASE | re.DOTALL, +) + + +def _normalize_extension(ext_value: Any) -> str: + """Sanitize extension strings to alphanumerics and cap at 5 chars.""" + return normalize_file_extension(ext_value) + + +def _normalize_host(value: Any) -> str: + """Normalize host names for matching/filtering.""" + host = str(value or "").strip().lower() + if host.startswith("www."): + host = host[4:] + if ":" in host: + host = host.split(":", 1)[0] + return host + + +def _normalize_space(text: Any) -> str: + return _WHITESPACE_RE.sub(" ", str(text or "")).strip() + + +def _url_matches_site(cls, url: str, site_host: str) -> bool: + """Return True when URL host is the requested site/subdomain.""" + try: + parsed = urlparse(str(url or "")) + host = _normalize_host(getattr(parsed, "hostname", "") or "") + except Exception: + return False + + target = _normalize_host(site_host) + if not host or not target: + return False + return host == target or host.endswith(f".{target}") + + +def _itertext_join(node: Any) -> str: + try: + return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()]) + except Exception: + return "" + + +def _html_fragment_to_text(fragment: Any) -> str: + text = _HTML_TAG_RE.sub(" ", str(fragment or "")) + return _html.unescape(text) + + +def _append_web_result( + cls, + items: List[Dict[str, str]], + seen_urls: set[str], + *, + site_host: str, + url_text: str, + title_text: str, + snippet_text: str, +) -> None: + url_clean = str(url_text or "").strip() + if not url_clean or not url_clean.startswith(("http://", "https://")): + return + if not _url_matches_site(url_clean, site_host): + return + if url_clean in seen_urls: + return + + seen_urls.add(url_clean) + items.append( + { + "url": url_clean, + "title": _normalize_space(title_text) or url_clean, + "snippet": _normalize_space(snippet_text), + } + ) + + +def _parse_web_results_with_fallback( + cls, + *, + html_text: str, + limit: int, + lxml_parser: Any, + regex_parser: Any, + fallback_when_empty: bool = False, +) -> List[Dict[str, str]]: + """Run an lxml-based parser with an optional regex fallback.""" + items: List[Dict[str, str]] = [] + seen_urls: set[str] = set() + should_run_regex = False + + try: + from lxml import html as lxml_html + + doc = lxml_html.fromstring(html_text or "") + lxml_parser(doc, items, seen_urls) + should_run_regex = fallback_when_empty and not items + except Exception: + should_run_regex = True + + if should_run_regex: + regex_parser(html_text or "", items, seen_urls) + + return items[:limit] + + +def _extract_duckduckgo_target_url(href: Any) -> str: + """Extract direct target URL from DuckDuckGo result links.""" + raw_href = str(href or "").strip() + if not raw_href: + return "" + + if raw_href.startswith("//"): + raw_href = f"https:{raw_href}" + + if raw_href.startswith("/"): + raw_href = f"https://duckduckgo.com{raw_href}" + + parsed = None + try: + parsed = urlparse(raw_href) + except Exception: + parsed = None + + try: + host = str(getattr(parsed, "hostname", "") or "").strip().lower() + except Exception: + host = "" + + if host.endswith("duckduckgo.com"): + try: + query = parse_qs(str(getattr(parsed, "query", "") or "")) + candidate = (query.get("uddg") or [""])[0] + if candidate: + return str(unquote(candidate)).strip() + except Exception: + pass + + return raw_href + + +def _extract_yahoo_target_url(href: Any) -> str: + """Extract direct target URL from Yahoo redirect links.""" + raw_href = str(href or "").strip() + if not raw_href: + return "" + + ru_match = _YAHOO_RU_RE.search(raw_href) + if ru_match: + try: + return str(unquote(ru_match.group(1))).strip() + except Exception: + pass + + try: + parsed = urlparse(raw_href) + query = parse_qs(str(getattr(parsed, "query", "") or "")) + candidate = (query.get("RU") or query.get("ru") or [""])[0] + if candidate: + return str(unquote(candidate)).strip() + except Exception: + pass + + return raw_href + + +def parse_duckduckgo_results( + cls, + *, + html_text: str, + site_host: str, + limit: int, +) -> List[Dict[str, str]]: + """Parse DuckDuckGo HTML results into normalized rows.""" + def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + result_nodes = doc.xpath("//div[contains(@class, 'result')]") + + for node in result_nodes: + links = node.xpath(".//a[contains(@class, 'result__a')]") + if not links: + continue + + link = links[0] + href = _extract_duckduckgo_target_url(link.get("href")) + title = _itertext_join(link) + + snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]") + snippet = "" + if snippet_nodes: + snippet = _itertext_join(snippet_nodes[0]) + + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text=snippet, + ) + if len(items) >= limit: + break + + def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html): + href = _extract_duckduckgo_target_url(match.group(1)) + title_html = match.group(2) + title = _html_fragment_to_text(title_html) + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text="", + ) + if len(items) >= limit: + break + + return _parse_web_results_with_fallback( + cls, + html_text=html_text, + limit=limit, + lxml_parser=_parse_lxml, + regex_parser=_parse_regex, + fallback_when_empty=True, + ) + + +def parse_yahoo_results( + cls, + *, + html_text: str, + site_host: str, + limit: int, +) -> List[Dict[str, str]]: + """Parse Yahoo HTML search results into normalized rows.""" + def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + for node in doc.xpath("//a[@href]"): + href = _extract_yahoo_target_url(node.get("href")) + title = _itertext_join(node) + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text="", + ) + if len(items) >= limit: + break + + def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + for match in _GENERIC_ANCHOR_RE.finditer(raw_html): + href = _extract_yahoo_target_url(match.group(1)) + title_html = match.group(2) + title = _html_fragment_to_text(title_html) + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text="", + ) + if len(items) >= limit: + break + + return _parse_web_results_with_fallback( + cls, + html_text=html_text, + limit=limit, + lxml_parser=_parse_lxml, + regex_parser=_parse_regex, + ) + + +def query_yahoo( + cls, + *, + search_query: str, + site_host: str, + limit: int, + session: Any, + deadline: Optional[float] = None, +) -> List[Dict[str, str]]: + """Fetch results from Yahoo search (robust fallback in bot-protected envs).""" + all_rows: List[Dict[str, str]] = [] + seen_urls: set[str] = set() + + max_pages = max(1, min((max(1, int(limit or 1)) + 9) // 10, 3)) + for page_idx in range(max_pages): + if deadline is not None and time.monotonic() >= deadline: + break + + params = { + "p": search_query, + "n": "10", + "b": str((page_idx * 10) + 1), + } + try: + read_timeout = 10.0 + if deadline is not None: + remaining = max(0.0, float(deadline - time.monotonic())) + if remaining <= 0.0: + break + read_timeout = max(3.0, min(10.0, remaining)) + + response = session.get( + "https://search.yahoo.com/search", + params=params, + timeout=(3, read_timeout), + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "Accept-Language": "en-US,en;q=0.9", + }, + ) + response.raise_for_status() + except Exception: + break + + page_rows = parse_yahoo_results( + cls, + html_text=response.text, + site_host=site_host, + limit=max(1, limit - len(all_rows)), + ) + new_rows = 0 + for row in page_rows: + url_value = str(row.get("url") or "").strip() + if not url_value or url_value in seen_urls: + continue + seen_urls.add(url_value) + all_rows.append(row) + new_rows += 1 + if len(all_rows) >= limit: + break + + if len(all_rows) >= limit or new_rows == 0: + break + + return all_rows[:limit] + + +def parse_bing_results( + cls, + *, + html_text: str, + site_host: str, + limit: int, +) -> List[Dict[str, str]]: + """Parse Bing HTML search results into normalized rows.""" + def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]") + + for node in result_nodes: + links = node.xpath(".//h2/a") + if not links: + continue + link = links[0] + href = str(link.get("href") or "").strip() + title = _itertext_join(link) + + snippet = "" + for sel in ( + ".//*[contains(@class,'b_caption')]//p", + ".//*[contains(@class,'b_snippet')]", + ".//p", + ): + snip_nodes = node.xpath(sel) + if snip_nodes: + snippet = _itertext_join(snip_nodes[0]) + break + + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text=snippet, + ) + if len(items) >= limit: + break + + def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None: + for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html): + href = match.group(1) + title = _html_fragment_to_text(match.group(2)) + _append_web_result( + cls, + items, + seen_urls, + site_host=site_host, + url_text=href, + title_text=title, + snippet_text="", + ) + if len(items) >= limit: + break + + return _parse_web_results_with_fallback( + cls, + html_text=html_text, + limit=limit, + lxml_parser=_parse_lxml, + regex_parser=_parse_regex, + ) + + +def query_bing( + cls, + *, + search_query: str, + site_host: str, + limit: int, + session: Any, + deadline: Optional[float] = None, +) -> List[Dict[str, str]]: + """Fetch results from Bing (supports filetype: and site: natively).""" + all_rows: List[Dict[str, str]] = [] + seen_urls: set[str] = set() + + page_start = 1 + pages_checked = 0 + max_pages = max(1, min((max(1, int(limit or 1)) + 49) // 50, 3)) + while len(all_rows) < limit and pages_checked < max_pages: + if deadline is not None and time.monotonic() >= deadline: + break + + params = {"q": search_query, "first": str(page_start), "count": "50"} + try: + read_timeout = 10.0 + if deadline is not None: + remaining = max(0.0, float(deadline - time.monotonic())) + if remaining <= 0.0: + break + read_timeout = max(3.0, min(10.0, remaining)) + + response = session.get( + "https://www.bing.com/search", + params=params, + timeout=(3, read_timeout), + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "Accept-Language": "en-US,en;q=0.9", + }, + ) + response.raise_for_status() + except Exception: + break + + page_rows = parse_bing_results( + cls, + html_text=response.text, + site_host=site_host, + limit=max(1, limit - len(all_rows)), + ) + new_rows = 0 + for row in page_rows: + url_value = str(row.get("url") or "").strip() + if not url_value or url_value in seen_urls: + continue + seen_urls.add(url_value) + all_rows.append(row) + new_rows += 1 + if len(all_rows) >= limit: + break + + if new_rows == 0 or len(all_rows) >= limit: + break + page_start += 50 + pages_checked += 1 + + return all_rows + + +def query_web_search( + cls, + *, + search_query: str, + site_host: str, + limit: int, +) -> List[Dict[str, str]]: + """Execute web search and return parsed result rows. + + Uses Yahoo first (works in environments where Bing/DDG HTML endpoints + are challenge-gated), then Bing, then DuckDuckGo. + """ + from API.requests_client import get_requests_session + + session = get_requests_session() + normalized_limit = max(1, min(int(limit or 1), 100)) + engine_deadline = time.monotonic() + 12.0 + + all_rows = query_yahoo( + cls, + search_query=search_query, + site_host=site_host, + limit=normalized_limit, + session=session, + deadline=engine_deadline, + ) + if all_rows: + return all_rows[:normalized_limit] + + all_rows = query_bing( + cls, + search_query=search_query, + site_host=site_host, + limit=normalized_limit, + session=session, + deadline=engine_deadline, + ) + if all_rows: + return all_rows[:normalized_limit] + + all_rows_ddg: List[Dict[str, str]] = [] + seen_urls: set[str] = set() + endpoints = [ + "https://html.duckduckgo.com/html/", + "https://duckduckgo.com/html/", + ] + for endpoint in endpoints: + if time.monotonic() >= engine_deadline: + break + max_offsets = min(3, max(1, (normalized_limit + 29) // 30)) + for page_idx in range(max_offsets): + if time.monotonic() >= engine_deadline: + break + offset = page_idx * 30 + params = {"q": search_query, "s": str(offset)} + remaining = max(0.0, float(engine_deadline - time.monotonic())) + if remaining <= 0.0: + break + read_timeout = max(3.0, min(10.0, remaining)) + response = session.get( + endpoint, + params=params, + timeout=(3, read_timeout), + headers={"Referer": "https://duckduckgo.com/"}, + ) + response.raise_for_status() + page_rows = parse_duckduckgo_results( + cls, + html_text=response.text, + site_host=site_host, + limit=max(1, normalized_limit - len(all_rows_ddg)), + ) + new_rows = 0 + for row in page_rows: + url_value = str(row.get("url") or "").strip() + if not url_value or url_value in seen_urls: + continue + seen_urls.add(url_value) + all_rows_ddg.append(row) + new_rows += 1 + if len(all_rows_ddg) >= normalized_limit: + break + if len(all_rows_ddg) >= normalized_limit or new_rows == 0: + break + if all_rows_ddg: + break + + return all_rows_ddg[:normalized_limit] + + +def _is_probable_html_path(path_value: str) -> bool: + """Return True when URL path likely points to an HTML page.""" + path = str(path_value or "").strip() + if not path: + return True + suffix = Path(path).suffix.lower() + if not suffix: + return True + return suffix in {".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".shtml", ".xhtml"} + + +def _extract_html_links(cls, *, html_text: str, base_url: str) -> List[str]: + """Extract absolute links from an HTML document.""" + links: List[str] = [] + seen: set[str] = set() + + def _add_link(raw_href: Any) -> None: + href = str(raw_href or "").strip() + if not href or href.startswith(("#", "javascript:", "mailto:")): + return + try: + absolute = urljoin(base_url, href) + parsed = urlparse(absolute) + except Exception: + return + if str(getattr(parsed, "scheme", "") or "").lower() not in {"http", "https"}: + return + clean = parsed._replace(fragment="").geturl() + if clean in seen: + return + seen.add(clean) + links.append(clean) + + try: + from lxml import html as lxml_html + + doc = lxml_html.fromstring(html_text or "") + for node in doc.xpath("//a[@href]"): + _add_link(node.get("href")) + except Exception: + href_pattern = re.compile(r'<a[^>]+href=["\']([^"\']+)["\']', flags=re.IGNORECASE) + for match in href_pattern.finditer(html_text or ""): + _add_link(match.group(1)) + + return links + + +def crawl_site_for_extension( + cls, + *, + seed_url: str, + site_host: str, + extension: str, + limit: int, + max_duration_seconds: float = 15.0, +) -> List[Dict[str, str]]: + """Fallback crawler that discovers in-site file links by extension.""" + from API.requests_client import get_requests_session + + normalized_ext = _normalize_extension(extension) + if not normalized_ext: + return [] + + start_url = _normalize_seed_url(cls, seed_url, site_host) + if not start_url: + return [] + + session = get_requests_session() + headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "Accept-Language": "en-US,en;q=0.9", + } + + queue: deque[str] = deque([start_url]) + queued: set[str] = {start_url} + visited_pages: set[str] = set() + seen_files: set[str] = set() + rows: List[Dict[str, str]] = [] + normalized_limit = max(1, min(int(limit or 1), 100)) + max_pages = max(8, min(normalized_limit * 4, 64)) + crawl_deadline = time.monotonic() + max(5.0, float(max_duration_seconds or 0.0)) + + while ( + queue + and len(visited_pages) < max_pages + and len(rows) < normalized_limit + and time.monotonic() < crawl_deadline + ): + page_url = queue.popleft() + queued.discard(page_url) + if page_url in visited_pages: + continue + visited_pages.add(page_url) + + if time.monotonic() >= crawl_deadline: + break + + try: + response = session.get(page_url, timeout=(4, 8), headers=headers) + response.raise_for_status() + except Exception: + continue + + final_url = str(getattr(response, "url", "") or page_url) + try: + parsed_final = urlparse(final_url) + except Exception: + continue + + final_host = _normalize_host(getattr(parsed_final, "hostname", "") or "") + if not _url_matches_site(final_url, site_host): + continue + + final_path = str(getattr(parsed_final, "path", "") or "") + direct_ext = _normalize_extension(Path(final_path).suffix) + if direct_ext == normalized_ext: + file_url = parsed_final._replace(fragment="").geturl() + if file_url not in seen_files: + seen_files.add(file_url) + title = Path(unquote(final_path)).name or file_url + rows.append( + { + "url": file_url, + "title": title, + "snippet": "Discovered via in-site crawl", + } + ) + continue + + content_type = str((response.headers or {}).get("content-type", "") or "").lower() + if "html" not in content_type and "xhtml" not in content_type: + continue + + html_text = str(getattr(response, "text", "") or "") + if not html_text: + continue + if len(html_text) > 2_500_000: + continue + + discovered_links = _extract_html_links(cls, html_text=html_text, base_url=final_url) + for idx, target in enumerate(discovered_links): + if len(rows) >= normalized_limit: + break + if idx >= 300: + break + if time.monotonic() >= crawl_deadline: + break + try: + parsed_target = urlparse(target) + except Exception: + continue + target_host = _normalize_host(getattr(parsed_target, "hostname", "") or "") + if not target_host or not (target_host == final_host or target_host.endswith(f".{site_host}")): + if not _url_matches_site(target, site_host): + continue + + target_clean = parsed_target._replace(fragment="").geturl() + target_path = str(getattr(parsed_target, "path", "") or "") + target_ext = _normalize_extension(Path(target_path).suffix) + + if target_ext == normalized_ext: + if target_clean in seen_files: + continue + seen_files.add(target_clean) + title = Path(unquote(target_path)).name or target_clean + rows.append( + { + "url": target_clean, + "title": title, + "snippet": f"Discovered via crawl from {final_path or '/'}", + } + ) + continue + + if _is_probable_html_path(target_path): + if target_clean not in visited_pages and target_clean not in queued: + queue.append(target_clean) + queued.add(target_clean) + + if time.monotonic() >= crawl_deadline: + debug( + "Web crawl fallback reached time budget", + { + "site": site_host, + "visited_pages": len(visited_pages), + "queued_pages": len(queue), + "results": len(rows), + "time_budget_seconds": max_duration_seconds, + }, + ) + + return rows[:normalized_limit] + + +def _extract_site_host(cls, candidate: Any) -> Optional[str]: + """Extract a host/domain from URL-like input.""" + raw = str(candidate or "").strip().strip('"').strip("'") + if not raw: + return None + + if raw.lower().startswith("site:"): + raw = raw.split(":", 1)[1].strip() + + parsed = None + try: + parsed = urlparse(raw) + except Exception: + parsed = None + + if parsed is None or not getattr(parsed, "hostname", None): + try: + parsed = urlparse(f"https://{raw}") + except Exception: + parsed = None + + host = "" + try: + host = str(getattr(parsed, "hostname", "") or "").strip().lower() + except Exception: + host = "" + + host = _normalize_host(host) + if not host or "." not in host: + return None + return host + + +def _normalize_seed_url(cls, seed_value: Any, site_host: str) -> str: + """Build a safe crawl starting URL from user input and resolved host.""" + raw = str(seed_value or "").strip().strip("'\"") + if not raw: + raw = str(site_host or "").strip() + + if raw and not raw.startswith(("http://", "https://")): + raw = f"https://{raw}" + + try: + parsed = urlparse(raw) + except Exception: + parsed = urlparse("") + + target = _normalize_host(site_host) + host = _normalize_host(getattr(parsed, "hostname", "") or "") + if target and host and not (host == target or host.endswith(f".{target}")): + return f"https://{target}/" + + scheme = str(getattr(parsed, "scheme", "") or "https").lower() + if scheme not in {"http", "https"}: + scheme = "https" + + netloc = str(getattr(parsed, "netloc", "") or "").strip() + if not netloc: + netloc = target + path = str(getattr(parsed, "path", "") or "").strip() + if not path: + path = "/" + + return f"{scheme}://{netloc}{path}" + + +def build_web_search_plan( + cls, + *, + query: str, + positional_args: List[str], + storage_backend: Optional[str], + store_filter: Optional[str], + hash_query: List[str], +) -> Optional[Dict[str, Any]]: + """Build web-search plan for URL + ext/filetype query syntax. + + Example input: + search-file "example.com/foo" -query "ext:pdf" + Produces: + site:example.com filetype:pdf + """ + if storage_backend or store_filter or hash_query: + return None + + text = _normalize_space(query) + if not text: + return None + + local_markers = ("url:", "hash:", "tag:", "instance:", "system:") + if any(marker in text.lower() for marker in local_markers): + return None + + site_host: Optional[str] = None + site_from_positional = False + site_token_to_strip = "" + seed_url = "" + + site_match = _SITE_TOKEN_RE.search(text) + if site_match: + site_host = _extract_site_host(cls, site_match.group(1)) + seed_url = str(site_match.group(1) or "").strip() + + if not site_host and positional_args: + site_host = _extract_site_host(cls, positional_args[0]) + site_from_positional = bool(site_host) + if site_from_positional: + site_token_to_strip = str(positional_args[0] or "").strip() + seed_url = site_token_to_strip + + if not site_host: + for token in text.split(): + candidate = str(token or "").strip().strip(",") + if not candidate: + continue + lower_candidate = candidate.lower() + if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")): + continue + if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith( + ("http://", "https://") + ): + continue + guessed = _extract_site_host(cls, candidate) + if guessed: + site_host = guessed + site_token_to_strip = candidate + break + + if not site_host: + return None + + filetype_match = _FILETYPE_TOKEN_RE.search(text) + filetype = _normalize_extension(filetype_match.group(1)) if filetype_match else "" + + has_explicit_site = bool(site_match) + if not filetype and not has_explicit_site: + return None + + residual = text + residual = _SITE_REMOVE_RE.sub(" ", residual) + residual = _FILETYPE_REMOVE_RE.sub(" ", residual) + + if site_from_positional and positional_args: + first = str(positional_args[0] or "").strip() + if first: + residual = re.sub(rf"(?:^|\s){re.escape(first)}(?:\s|$)", " ", residual, count=1) + elif site_token_to_strip: + residual = re.sub( + rf"(?:^|\s){re.escape(site_token_to_strip)}(?:\s|$)", + " ", + residual, + count=1, + ) + + residual = _normalize_space(residual) + + search_terms: List[str] = [f"site:{site_host}"] + if filetype: + search_terms.append(f"filetype:{filetype}") + if residual: + search_terms.append(residual) + + search_query = " ".join(search_terms).strip() + if not search_query: + return None + + normalized_seed_url = _normalize_seed_url(cls, seed_url, site_host) + + return { + "site_host": site_host, + "filetype": filetype, + "search_query": search_query, + "residual": residual, + "seed_url": normalized_seed_url, + } diff --git a/cmdlet/get_url.py b/cmdlet/get_url.py index 5db02aa..e76928e 100644 --- a/cmdlet/get_url.py +++ b/cmdlet/get_url.py @@ -475,7 +475,7 @@ class Get_Url(Cmdlet): publish_result_table(ctx, table if display_items else None, display_items, subject=result) - # Emit after table state is finalized to prevent side effects in TUI rendering + # Emit after table state is finalized for d in display_items: ctx.emit(d) diff --git a/cmdlet/metadata/relationship_add.py b/cmdlet/metadata/relationship_add.py index e861e40..0a7fbee 100644 --- a/cmdlet/metadata/relationship_add.py +++ b/cmdlet/metadata/relationship_add.py @@ -60,8 +60,7 @@ CMDLET = Cmdlet( "- Mode 2: Use -king to explicitly set which item/hash is the king: @1-3 | add-relationship -king @4", "- Mode 2b: Use -king and -alt to select both sides from the last table: add-relationship -king @1 -alt @3-5", "- Mode 3: Read relationships from sidecar tags:", - " - New format: 'relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>' (first hash is king)", - " - Legacy: 'relationship: hash(king)<HASH>,hash(alt)<HASH>...'", + " - Format: 'relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>' (first hash is king)", "- Supports three relationship types: king (primary), alt (alternative), related (other versions)", "- When using -king, all piped items become the specified relationship type to the king", ], @@ -71,13 +70,10 @@ CMDLET = Cmdlet( _normalize_hash_hex = sh.normalize_hash -def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]: + def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]: """Parse relationship tags. - Supported formats: - - New: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH> - - Old: relationship: hash(king)<HASH>,hash(alt)<HASH>... - + Format: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH> Returns a dict like {"king": ["HASH1"], "alt": ["HASH2"], ...} """ result: Dict[str, @@ -85,20 +81,7 @@ def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]: if not isinstance(tag_value, str): return result - # Match patterns like hash(king)HASH or hash(type)<HASH> - pattern = r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?" - matches = re.findall(pattern, tag_value) - - if matches: - for rel_type, hash_value in matches: - normalized = _normalize_hash_hex(hash_value) - if normalized: - if rel_type not in result: - result[rel_type] = [] - result[rel_type].append(normalized) - return result - - # New format: extract hashes, first is king + # Extract hashes; first is king hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", tag_value) hashes = [h.strip().lower() for h in hashes if isinstance(h, str)] if not hashes: @@ -833,7 +816,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: hydrus_client.set_relationship(h, king_hash, str(rel_type)) return 0 - # Process each item in the list (legacy path-based mode) + # Process each item in the list (path-based mode) for item in items_to_process: # Extract hash and path from current item file_hash = None @@ -1033,136 +1016,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: return 0 - # FILE MODE: Read relationships from sidecar (legacy mode - for -path arg only) - log( - "Note: Use piping mode for easier relationships. Example: 1,2,3 | add-relationship", - file=sys.stderr, - ) - - # Resolve media path from -path arg or result target - target = getattr(result, "target", None) or getattr(result, "path", None) - media_path = ( - arg_path - if arg_path is not None else Path(str(target)) if isinstance(target, - str) else None - ) - if media_path is None: - log("Provide -path <file> or pipe a local file result", file=sys.stderr) - return 1 - - # Validate local file - if str(media_path).lower().startswith(("http://", "https://")): - log("This cmdlet requires a local file path, not a URL", file=sys.stderr) - return 1 - if not media_path.exists() or not media_path.is_file(): - log(f"File not found: {media_path}", file=sys.stderr) - return 1 - - # Build Hydrus client - hydrus_provider = get_plugin("hydrusnetwork", config) - try: - hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None - except Exception as exc: - log(f"Hydrus client unavailable: {exc}", file=sys.stderr) - return 1 - - if hydrus_client is None: - log("Hydrus client unavailable", file=sys.stderr) - return 1 - - # Read sidecar to find relationship tags - sidecar_path = find_sidecar(media_path) - if sidecar_path is None: - log(f"No sidecar found for {media_path.name}", file=sys.stderr) - return 1 - - try: - _, tags, _ = read_sidecar(sidecar_path) - except Exception as exc: - log(f"Failed to read sidecar: {exc}", file=sys.stderr) - return 1 - - # Find relationship tags (format: "relationship: hash(king)<HASH>,hash(alt)<HASH>,hash(related)<HASH>") - relationship_tags = [ - t for t in tags if isinstance(t, str) and t.lower().startswith("relationship:") - ] - - if not relationship_tags: - log("No relationship tags found in sidecar", file=sys.stderr) - return 0 # Not an error, just nothing to do - - # Get the file hash from result (should have been set by add-file) - file_hash = getattr(result, "hash_hex", None) - if not file_hash: - log("File hash not available (run add-file first)", file=sys.stderr) - return 1 - - file_hash = _normalize_hash_hex(file_hash) - if not file_hash: - log("Invalid file hash format", file=sys.stderr) - return 1 - - # Parse relationships from tags and apply them - success_count = 0 - error_count = 0 - - for rel_tag in relationship_tags: - try: - # Parse: "relationship: hash(king)<HASH>,hash(alt)<HASH>,hash(related)<HASH>" - rel_str = rel_tag.split(":", 1)[1].strip() # Get part after "relationship:" - - # Parse relationships - rels = _extract_relationships_from_tag(f"relationship: {rel_str}") - - # Set the relationships in Hydrus - for rel_type, related_hashes in rels.items(): - if not related_hashes: - continue - - for related_hash in related_hashes: - # Don't set relationship between hash and itself - if file_hash == related_hash: - continue - - try: - hydrus_client.set_relationship( - file_hash, - related_hash, - rel_type - ) - log( - f"[add-relationship] Set {rel_type} relationship: " - f"{file_hash} <-> {related_hash}", - file=sys.stderr, - ) - success_count += 1 - except Exception as exc: - log( - f"Failed to set {rel_type} relationship: {exc}", - file=sys.stderr - ) - error_count += 1 - - except Exception as exc: - log(f"Failed to parse relationship tag: {exc}", file=sys.stderr) - error_count += 1 - - if success_count > 0: - log( - f"Successfully set {success_count} relationship(s) for {media_path.name}", - file=sys.stderr, - ) - ctx.emit( - f"add-relationship: {media_path.name} ({success_count} relationships set)" - ) - return 0 - elif error_count == 0: - log("No relationships to set", file=sys.stderr) - return 0 # Success with nothing to do - else: - log(f"Failed with {error_count} error(s)", file=sys.stderr) - return 1 - # Register cmdlet (no legacy decorator) CMDLET.exec = _run diff --git a/cmdlet/metadata/tag_add.py b/cmdlet/metadata/tag_add.py index b06927c..de0a12b 100644 --- a/cmdlet/metadata/tag_add.py +++ b/cmdlet/metadata/tag_add.py @@ -1081,6 +1081,8 @@ class Add_Tag(Cmdlet): ok_add = True if not queued_bulk: + ok_add = False + ok_remove = True try: ok_add = backend.add_tag( resolved_hash, @@ -1088,11 +1090,19 @@ class Add_Tag(Cmdlet): config=config, existing_tags=existing_tag_list, ) - if not ok_add: - log("[add_tag] Warning: Store rejected tag update", file=sys.stderr) except Exception as exc: log(f"[add_tag] Warning: Failed adding tag: {exc}", file=sys.stderr) ok_add = False + if tags_to_remove: + try: + delete_fn = getattr(backend, "delete_tag", None) + if callable(delete_fn): + ok_remove = delete_fn(resolved_hash, list(tags_to_remove), config=config) + except Exception as exc: + log(f"[add_tag] Warning: Failed removing tag: {exc}", file=sys.stderr) + ok_remove = False + if not ok_add and not ok_remove: + log("[add_tag] Warning: Store rejected tag update", file=sys.stderr) if ok_add and merged_tags: refreshed_list = list(merged_tags) diff --git a/cmdnat/config.py b/cmdnat/config.py index 67b43fd..f916847 100644 --- a/cmdnat/config.py +++ b/cmdnat/config.py @@ -1158,8 +1158,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: if not args: if sys.stdin.isatty() and not piped_result: print( - "Interactive TUI config editor has been discontinued. " - "Showing configuration table instead." + "Run `.config <key>` to view or edit a specific setting. " ) return _show_config_table(current_config) diff --git a/cmdnat/status.py b/cmdnat/status.py index 51a66dc..ee51a2e 100644 --- a/cmdnat/status.py +++ b/cmdnat/status.py @@ -60,8 +60,7 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int: _add_startup_check(startup_table, "ERROR", "STATUS", detail=str(exc)) if startup_table.rows: - # Mark as rendered to prevent CLI.py from auto-printing it to stdout - # (avoiding duplication in TUI logs, while keeping it in TUI Results) + # Mark as rendered to prevent CLI.py from auto-printing it to stdout setattr(startup_table, "_rendered_by_cmdlet", True) ctx.set_current_stage_table(startup_table) diff --git a/plugins/local/__init__.py b/plugins/local/__init__.py index c280f3c..776c414 100644 --- a/plugins/local/__init__.py +++ b/plugins/local/__init__.py @@ -4,11 +4,27 @@ import shutil from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from PluginCore.base import Provider -from SYS.metadata import write_metadata, write_tags +from PluginCore.base import Provider, SearchResult +from SYS.metadata import _read_sidecar_metadata, read_tags_from_file, write_metadata, write_tags from SYS.utils import sanitize_filename, sha256_file, unique_path +def _format_size_safe(size_bytes: Any) -> str: + if size_bytes is None: + return "" + try: + size = int(size_bytes) + except (TypeError, ValueError): + return str(size_bytes or "") + if size < 1024: + return f"{size} B" + if size < 1024 * 1024: + return f"{size / 1024:.1f} KB" + if size < 1024 * 1024 * 1024: + return f"{size / (1024 * 1024):.1f} MB" + return f"{size / (1024 * 1024 * 1024):.2f} GB" + + def _copy_sidecars(source_path: Path, target_path: Path) -> None: possible_sidecars = [ source_path.with_suffix(source_path.suffix + ".json"), @@ -80,7 +96,7 @@ class Local(Provider): PLUGIN_NAME = "local" PLUGIN_ALIASES = ("filesystem", "fs") MULTI_INSTANCE = True - SUPPORTED_CMDLETS = frozenset({"add-file"}) + SUPPORTED_CMDLETS = frozenset({"add-file", "search-file"}) @property def label(self) -> str: @@ -165,6 +181,152 @@ class Local(Provider): def validate(self) -> bool: return True + @staticmethod + def _infer_media_kind(ext: str) -> str: + e = str(ext or "").strip().lower().lstrip(".") + if e in {"mp3", "flac", "wav", "ogg", "opus", "m4a", "aac", "wma", "aiff"}: + return "audio" + if e in {"mp4", "mkv", "avi", "mov", "webm", "wmv", "flv", "m4v"}: + return "video" + if e in {"pdf", "epub", "mobi", "azw3", "djvu", "cbr", "cbz", "chm"}: + return "book" + if e in {"zip", "rar", "7z", "tar", "gz", "bz2", "xz", "iso"}: + return "archive" + if e in {"exe", "msi", "apk", "deb", "rpm", "appimage", "bin"}: + return "software" + if e in {"jpg", "jpeg", "png", "gif", "bmp", "webp", "svg", "tiff", "ico"}: + return "image" + return "file" + + def search( + self, + query: str, + limit: int = 50, + filters: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> List[SearchResult]: + q = str(query or "").strip() + match_all = not q or q == "*" + query_tokens = [t.lower() for t in q.split()] if not match_all else [] + max_results = max(1, int(limit)) + + target_instance = None + if filters: + target_instance = str(filters.get("instance") or filters.get("store") or "").strip() or None + + results: List[SearchResult] = [] + instances = self.plugin_instance_configs() + + for inst_name, inst_cfg in instances.items(): + if target_instance and str(inst_name or "").strip().lower() != target_instance.lower(): + continue + + settings = self._settings_from_config(inst_cfg, instance_name=inst_name) + root_text = str(settings.get("path") or "").strip() + if not root_text: + continue + + root = Path(root_text).expanduser() + if not root.is_dir(): + continue + + try: + for entry in root.rglob("*"): + if len(results) >= max_results: + break + if not entry.is_file(): + continue + + file_name = entry.name + file_stem = entry.stem + + tag_path = entry.parent / (file_name + ".tag") + meta_path = entry.parent / (file_name + ".metadata") + + has_tag = tag_path.is_file() + has_meta = meta_path.is_file() + if not has_tag and not has_meta: + continue + + tags: List[str] = [] + if has_tag: + try: + tags = read_tags_from_file(tag_path) + except Exception: + tags = [] + + hash_value: Optional[str] = None + urls: List[str] = [] + if has_meta: + try: + hash_value, _extra_tags, urls = _read_sidecar_metadata(meta_path) + except Exception: + pass + + if not match_all and query_tokens: + url_text = " ".join(urls) if urls else "" + search_text = " ".join([file_name.lower(), file_stem.lower().replace("_", " "), *tags, url_text]) + if hash_value: + search_text += " " + hash_value.lower() + if not all(token in search_text for token in query_tokens): + continue + + try: + size_bytes = int(entry.stat().st_size) + except Exception: + size_bytes = None + + ext = entry.suffix.lstrip(".") + media_kind = self._infer_media_kind(ext) + + inst_label = str(inst_name or "").strip() + if inst_label.lower() == "default": + inst_label = "" + + store_label = f"local:{inst_label}" if inst_label else "local" + tag_text = ", ".join(tags[:8]) if tags else "" + + metadata: Dict[str, Any] = { + "store": store_label, + "ext": ext, + "size": size_bytes, + } + if inst_label: + metadata["instance"] = inst_label + if hash_value: + metadata["hash"] = hash_value + if urls: + metadata["url"] = urls + + sr = SearchResult( + table="local", + title=file_name, + path=str(entry), + detail=store_label, + annotations=[store_label], + media_kind=media_kind, + size_bytes=size_bytes, + tag=set(tags), + columns=[ + ("Title", file_name), + ("Tag", tag_text), + ("Store", store_label), + ("Size", _format_size_safe(size_bytes)), + ("Ext", ext), + ], + full_metadata=metadata, + ) + sr.ext = ext + sr.size = size_bytes + if hash_value: + sr.hash = hash_value + results.append(sr) + + except Exception: + continue + + return results[:max_results] + @staticmethod def _folder_name_from_pipe(pipe_obj: Any) -> str: metadata = getattr(pipe_obj, "metadata", None) diff --git a/plugins/mpv/commands.py b/plugins/mpv/commands.py index c676f8d..65121d4 100644 --- a/plugins/mpv/commands.py +++ b/plugins/mpv/commands.py @@ -1,5 +1,6 @@ # pyright: reportUnusedFunction=false from typing import Any, Dict, Sequence, List, Optional +import atexit import os import sys import json @@ -21,6 +22,8 @@ from SYS.config import get_hydrus_access_key, get_hydrus_url, resolve_cookies_pa _NOTES_PREFETCH_INFLIGHT: set[str] = set() _NOTES_PREFETCH_LOCK = threading.Lock() +_PREFETCH_THREADS: set[threading.Thread] = set() +_PREFETCH_THREADS_LOCK = threading.Lock() _PLAYLIST_STORE_CACHE: Optional[Dict[str, Any]] = None _PLAYLIST_STORE_MTIME_NS: Optional[int] = None _SHA256_RE = re.compile(r"[0-9a-f]{64}") @@ -347,21 +350,20 @@ def _get_latest_mpv_run_marker(log_db_path: str) -> Optional[tuple[str, str]]: try: import sqlite3 - conn = sqlite3.connect(log_db_path, timeout=5.0) - cur = conn.cursor() - cur.execute( - ( - "SELECT timestamp, message FROM logs " - "WHERE module = 'mpv' AND (" - "message LIKE '[helper] version=% started ipc=%' " - "OR message LIKE 'medeia lua loaded version=%'" - ") " - "ORDER BY timestamp DESC LIMIT 1" + with sqlite3.connect(log_db_path, timeout=5.0) as conn: + cur = conn.cursor() + cur.execute( + ( + "SELECT timestamp, message FROM logs " + "WHERE module = 'mpv' AND (" + "message LIKE '[helper] version=% started ipc=%' " + "OR message LIKE 'medeia lua loaded version=%'" + ") " + "ORDER BY timestamp DESC LIMIT 1" + ) ) - ) - row = cur.fetchone() - cur.close() - conn.close() + row = cur.fetchone() + cur.close() if row and row[0]: return str(row[0]), str(row[1] or "") except Exception: @@ -381,26 +383,25 @@ def _get_mpv_logs_for_latest_run( try: import sqlite3 - conn = sqlite3.connect(log_db_path, timeout=5.0) - cur = conn.cursor() - query = "SELECT timestamp, level, module, message FROM logs WHERE module = 'mpv'" - params: List[str] = [] - if marker_ts: - query += " AND timestamp >= ?" - params.append(marker_ts) - else: - cutoff = (datetime.utcnow() - timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S") - query += " AND timestamp >= ?" - params.append(cutoff) - if log_filter_text: - query += " AND LOWER(message) LIKE ?" - params.append(f"%{log_filter_text.lower()}%") - query += " ORDER BY timestamp DESC LIMIT ?" - params.append(str(max(1, int(limit)))) - cur.execute(query, tuple(params)) - rows = cur.fetchall() - cur.close() - conn.close() + with sqlite3.connect(log_db_path, timeout=5.0) as conn: + cur = conn.cursor() + query = "SELECT timestamp, level, module, message FROM logs WHERE module = 'mpv'" + params: List[str] = [] + if marker_ts: + query += " AND timestamp >= ?" + params.append(marker_ts) + else: + cutoff = (datetime.utcnow() - timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S") + query += " AND timestamp >= ?" + params.append(cutoff) + if log_filter_text: + query += " AND LOWER(message) LIKE ?" + params.append(f"%{log_filter_text.lower()}%") + query += " ORDER BY timestamp DESC LIMIT ?" + params.append(str(max(1, int(limit)))) + cur.execute(query, tuple(params)) + rows = cur.fetchall() + cur.close() rows.reverse() return marker_ts, marker_msg, rows except Exception: @@ -849,11 +850,26 @@ def _prefetch_notes_async( thread = threading.Thread( target=_worker, name=f"mpv-notes-prefetch-{file_hash[:8]}", - daemon=True, ) + with _PREFETCH_THREADS_LOCK: + _PREFETCH_THREADS.add(thread) thread.start() +def _shutdown_prefetch_threads() -> None: + with _PREFETCH_THREADS_LOCK: + threads = list(_PREFETCH_THREADS) + _PREFETCH_THREADS.clear() + for t in threads: + try: + t.join(timeout=1.0) + except Exception: + pass + + +atexit.register(_shutdown_prefetch_threads) + + def _schedule_notes_prefetch(items: Sequence[Any], config: Optional[Dict[str, Any]]) -> None: limit = _get_lyric_prefetch_limit(config) if limit <= 0: diff --git a/plugins/mpv/format_probe.py b/plugins/mpv/format_probe.py index ec7b680..abef49a 100644 --- a/plugins/mpv/format_probe.py +++ b/plugins/mpv/format_probe.py @@ -14,11 +14,6 @@ def _repo_root() -> Path: return package_dir.parent -REPO_ROOT = _repo_root() -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - - def main(argv: list[str] | None = None) -> int: args = list(sys.argv[1:] if argv is None else argv) if not args: @@ -35,10 +30,19 @@ def main(argv: list[str] | None = None) -> int: url = str(args[0] or "").strip() captured_stdout = io.StringIO() captured_stderr = io.StringIO() - with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr): - from plugins.mpv.pipeline_helper import _run_op - payload = _run_op("ytdlp-formats", {"url": url}) + _root_str = str(_repo_root()) + _path_added = _root_str not in sys.path + if _path_added: + sys.path.insert(0, _root_str) + try: + with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr): + from plugins.mpv.pipeline_helper import _run_op + + payload = _run_op("ytdlp-formats", {"url": url}) + finally: + if _path_added: + sys.path.remove(_root_str) noisy_stdout = captured_stdout.getvalue().strip() noisy_stderr = captured_stderr.getvalue().strip() diff --git a/plugins/mpv/mpv_ipc.py b/plugins/mpv/mpv_ipc.py index 10a9bf4..f2e5597 100644 --- a/plugins/mpv/mpv_ipc.py +++ b/plugins/mpv/mpv_ipc.py @@ -7,6 +7,7 @@ This is the central hub for all Python-mpv IPC communication. The Lua script should use the Python CLI, which uses this module to manage mpv connections. """ +import atexit import ctypes import json import os @@ -901,6 +902,7 @@ class MPVIPCClient: self.is_windows = platform.system() == "Windows" self.silent = bool(silent) self._recv_buffer: bytes = b"" + atexit.register(self.disconnect) def _write_payload(self, payload: str) -> None: if not self.sock: @@ -1229,10 +1231,6 @@ class MPVIPCClient: self.sock = None self._recv_buffer = b"" - def __del__(self) -> None: - """Cleanup on object destruction.""" - self.disconnect() - def __enter__(self): """Context manager entry.""" self.connect() diff --git a/plugins/mpv/pipeline_helper.py b/plugins/mpv/pipeline_helper.py index 16389e5..77f17ae 100644 --- a/plugins/mpv/pipeline_helper.py +++ b/plugins/mpv/pipeline_helper.py @@ -61,17 +61,22 @@ def _runtime_config_root() -> Path: # Make repo-local packages importable even when mpv starts us from another cwd. -_ROOT = str(_repo_root()) -if _ROOT not in sys.path: - sys.path.insert(0, _ROOT) - -from plugins.mpv.mpv_ipc import MPVIPCClient, _windows_kill_pids, _windows_hidden_subprocess_kwargs, _windows_list_mpv_pids # noqa: E402 -from SYS.config import load_config, reload_config # noqa: E402 -from SYS.logger import set_debug, debug, set_thread_stream # noqa: E402 -from SYS.repl_queue import enqueue_repl_command, repl_state_is_alive # noqa: E402 -from SYS.utils import format_bytes # noqa: E402 -from PluginCore.registry import get_plugin, get_plugin_class # noqa: E402 -from plugins.ytdlp.tooling import get_display_format_id, get_selection_format_id # noqa: E402 +_root_str = str(_repo_root()) +_path_added = _root_str not in sys.path +if _path_added: + sys.path.insert(0, _root_str) +try: + from plugins.mpv.mpv_ipc import MPVIPCClient, _windows_kill_pids, _windows_hidden_subprocess_kwargs, _windows_list_mpv_pids # noqa: E402 + from SYS.config import load_config, reload_config # noqa: E402 + from SYS.logger import set_debug, debug, set_thread_stream # noqa: E402 + from SYS.repl_queue import enqueue_repl_command, repl_state_is_alive # noqa: E402 + from SYS.utils import format_bytes # noqa: E402 + from PluginCore.registry import get_plugin, get_plugin_class # noqa: E402 + from plugins.ytdlp.tooling import get_display_format_id, get_selection_format_id # noqa: E402 +finally: + if _path_added: + sys.path.remove(_root_str) + del _root_str, _path_added REQUEST_PROP = "user-data/medeia-pipeline-request" RESPONSE_PROP = "user-data/medeia-pipeline-response" diff --git a/plugins/playwright/__init__.py b/plugins/playwright/__init__.py index e563772..cd1fb8c 100644 --- a/plugins/playwright/__init__.py +++ b/plugins/playwright/__init__.py @@ -7,7 +7,10 @@ import `plugins.playwright` even when Playwright itself is not installed. from __future__ import annotations +from .runtime import USER_AGENT + __all__ = [ + "USER_AGENT", "PlaywrightTimeoutError", "PlaywrightTool", "PlaywrightDefaults", diff --git a/plugins/playwright/runtime.py b/plugins/playwright/runtime.py index 77ead33..4ea1178 100644 --- a/plugins/playwright/runtime.py +++ b/plugins/playwright/runtime.py @@ -65,15 +65,18 @@ def _find_filename_from_cd(cd: str) -> Optional[str]: return None +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" +) + + @dataclass(slots=True) class PlaywrightDefaults: browser: str = "chromium" # chromium|firefox|webkit headless: bool = True - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/120.0.0.0 Safari/537.36" - ) + user_agent: str = USER_AGENT viewport_width: int = 1920 viewport_height: int = 1080 navigation_timeout_ms: int = 90_000 diff --git a/plugins/soulseek/__init__.py b/plugins/soulseek/__init__.py index 83ad440..9d22102 100644 --- a/plugins/soulseek/__init__.py +++ b/plugins/soulseek/__init__.py @@ -243,9 +243,11 @@ class _LineFilterStream(io.TextIOBase): def _suppress_aioslsk_noise() -> Any: """Temporarily suppress known aioslsk noise printed to stdout/stderr. - Opt out by setting DOWNLOW_SOULSEEK_VERBOSE=1. + Opt out by setting DOWNLOAD_SOULSEEK_VERBOSE=1. + + Note: The old env var name DOWNLOW_SOULSEEK_VERBOSE (typo) is deprecated. """ - if os.environ.get("DOWNLOW_SOULSEEK_VERBOSE"): + if os.environ.get("DOWNLOAD_SOULSEEK_VERBOSE") or os.environ.get("DOWNLOW_SOULSEEK_VERBOSE"): yield return @@ -404,7 +406,7 @@ class Soulseek(Provider): ) except RuntimeError: - # If we're already inside an event loop (e.g., TUI), fall back to a + # If we're already inside an event loop, fall back to a # dedicated loop in this thread. loop = asyncio.new_event_loop() try: @@ -568,6 +570,21 @@ class Soulseek(Provider): timeout=9.0, limit=limit) ) + except RuntimeError: + loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(loop) + flat_results, search_summary = loop.run_until_complete( + self.perform_search(query, timeout=9.0, limit=limit) + ) + finally: + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + except Exception: + pass + loop.close() + + try: if not flat_results: return [] diff --git a/plugins/telegram/__init__.py b/plugins/telegram/__init__.py index 856b01e..5990eb9 100644 --- a/plugins/telegram/__init__.py +++ b/plugins/telegram/__init__.py @@ -319,7 +319,7 @@ class Telegram(Provider): def _run_async_blocking(self, coro): """Run an awaitable to completion using a fresh event loop. - If an event loop is already running in this thread (common in REPL/TUI), + If an event loop is already running in this thread (common in REPL), runs the coroutine in a worker thread with its own loop. """ result: Dict[str, @@ -358,6 +358,13 @@ class Telegram(Provider): except Exception: pass finally: + try: + try: + loop.run_until_complete(loop.shutdown_default_executor()) + except Exception: + pass + except Exception: + pass try: loop.close() except Exception: @@ -559,6 +566,10 @@ class Telegram(Provider): asyncio.set_event_loop(loop) loop.run_until_complete(_auth_async()) finally: + try: + loop.run_until_complete(loop.shutdown_default_executor()) + except Exception: + pass try: loop.close() except Exception: diff --git a/plugins/ytdlp/__init__.py b/plugins/ytdlp/__init__.py index 4dc777d..f62bdc1 100644 --- a/plugins/ytdlp/__init__.py +++ b/plugins/ytdlp/__init__.py @@ -510,6 +510,14 @@ class ytdlp(TablePluginMixin, Provider): SEARCH_QUERY_KEYS = ("search", "q") SUPPORTED_CMDLETS = frozenset({"download-file", "search-file"}) + QUERY_ARG_CHOICES = { + "format": ["audio", "1080", "720", "480", "360", "best", "worst", "bestvideo+bestaudio"], + "clip": ["0:00-1:00"], + "audio": ["true", "false", "1", "0"], + "item": ["1"], + } + INLINE_QUERY_FIELD_CHOICES = QUERY_ARG_CHOICES + @staticmethod def config_schema() -> List[Dict[str, Any]]: return _ytdlp_config_schema() diff --git a/scripts/pyproject.toml b/pyproject.toml similarity index 90% rename from scripts/pyproject.toml rename to pyproject.toml index 406ebe1..b8eb4af 100644 --- a/scripts/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "Comprehensive media management and search platform with support f requires-python = ">=3.9,<3.14" license = {text = "MIT"} authors = [ - {name = "Your Name", email = "your.email@example.com"} + {name = "Nose", email = "goyimnose@nothing.blah"} ] keywords = ["media", "search", "management", "hydrus", "download", "cli", "tui"] classifiers = [ @@ -31,6 +31,7 @@ classifiers = [ dependencies = [ # Core CLI and TUI frameworks "typer>=0.9.0", + "rich>=13.7.0", "prompt-toolkit>=3.0.0", "textual>=0.30.0", @@ -41,12 +42,14 @@ dependencies = [ "charset-normalizer>=3.2.0", "certifi>=2024.12.0", "httpx>=0.25.0", + "internetarchive>=4.1.0", # Document and data handling "pypdf>=3.0.0", "mutagen>=1.46.0", "cbor2>=4.0", "zstandard>=0.23.0", + "pypandoc-binary", # Image and media support "Pillow>=10.0.0", @@ -109,10 +112,10 @@ mm = "scripts.cli_entry:main" medeia = "scripts.cli_entry:main" [project.urls] -Homepage = "https://github.com/yourusername/medeia-macina" -Documentation = "https://medeia-macina.readthedocs.io" -Repository = "https://github.com/yourusername/medeia-macina.git" -Issues = "https://github.com/yourusername/medeia-macina/issues" +Homepage = "https://code.glowers.club/goyimnose/Medios-Macina" +Documentation = "https://code.glowers.club/goyimnose/Medios-Macina" +Repository = "https://code.glowers.club/goyimnose/Medios-Macina.git" +Issues = "https://code.glowers.club/goyimnose/Medios-Macina/issues" [tool.setuptools] packages = [ @@ -123,7 +126,7 @@ packages = [ "PluginCore", "SYS", ] -package-dir = {"" = ".."} +package-dir = {"" = "."} [tool.setuptools.package-data] scripts = ["*.py"] diff --git a/readme.md b/readme.md index 3e7d56c..f44a249 100644 --- a/readme.md +++ b/readme.md @@ -41,9 +41,7 @@ See [plugins/README.md](plugins/README.md) for plugin packaging and discovery de ## Configuration -The old interactive TUI config editor has been discontinued. - -Use `.config` from inside the CLI instead: +Use `.config` from inside the CLI: - `.config` opens the root configuration table. - `@N` drills into a selected section. diff --git a/scripts/_debug_token_windows_path.py b/scripts/_debug_token_windows_path.py new file mode 100644 index 0000000..1d5fbeb --- /dev/null +++ b/scripts/_debug_token_windows_path.py @@ -0,0 +1,10 @@ +from CLI import MedeiaLexer + + +class DummyDocument: + def __init__(self, lines): + self.lines = list(lines) + +lexer = MedeiaLexer() +doc = DummyDocument([r'C:\path\to\file']) +print(lexer.lex_document(doc)(0)) diff --git a/scripts/_run_cli_parsing_manual.py b/scripts/_run_cli_parsing_manual.py new file mode 100644 index 0000000..ee036ba --- /dev/null +++ b/scripts/_run_cli_parsing_manual.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import sys +import traceback + +sys.path.insert(0, r'C:\Forgejo\Medios-Macina') +from tests.test_cli_parsing import * + +failed = 0 +tests = [(n, o) for n, o in globals().items() if n.startswith('test_') and callable(o)] +for n, o in tests: + try: + o() + print(n, 'OK') + except Exception as e: + print(n, 'FAIL', e) + traceback.print_exc() + failed += 1 + +if failed: + sys.exit(1) +print('All tests passed') diff --git a/scripts/check_imports.py b/scripts/check_imports.py new file mode 100644 index 0000000..5c6dad6 --- /dev/null +++ b/scripts/check_imports.py @@ -0,0 +1,10 @@ +import importlib +import sys +import traceback + +try: + importlib.import_module("CLI") + print("CLI imported OK") +except Exception: + traceback.print_exc() + sys.exit(1) diff --git a/scripts/check_indentation.py b/scripts/check_indentation.py new file mode 100644 index 0000000..91fe493 --- /dev/null +++ b/scripts/check_indentation.py @@ -0,0 +1,23 @@ +from pathlib import Path +p = Path('Store/registry.py') +lines = p.read_text(encoding='utf-8').splitlines() +stack = [0] +for i, line in enumerate(lines, start=1): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(' ')) + if leading > stack[-1]: + # increased indent -> push + stack.append(leading) + else: + # dedent: must match an existing level + if leading != stack[-1]: + if leading in stack: + # pop until match + while stack and stack[-1] != leading: + stack.pop() + else: + print(f"Line {i}: inconsistent indent {leading}, stack levels {stack}") + print(repr(line)) + break +print('Done') \ No newline at end of file diff --git a/scripts/check_pattern.py b/scripts/check_pattern.py new file mode 100644 index 0000000..dd3849c --- /dev/null +++ b/scripts/check_pattern.py @@ -0,0 +1,19 @@ +import re +from pathlib import Path +p = Path(r'c:\Forgejo\Medios-Macina\CLI.py') +s = p.read_text(encoding='utf-8') +pattern = re.compile(r'(?s)if False:\s*class _OldPipelineExecutor:.*?from rich\\.markdown import Markdown\\s*') +m = pattern.search(s) +print('found', bool(m)) +if m: + print('start', m.start(), 'end', m.end()) + print('snippet:', s[m.start():m.start()+120]) +else: + # print a slice around the if False for debugging + i = s.find('if False:') + print('if False index', i) + print('around if False:', s[max(0,i-50):i+200]) + j = s.find('from rich.markdown import Markdown', i) + print('next from rich index after if False', j) + if j!=-1: + print('around that:', s[j-50:j+80]) diff --git a/scripts/check_store.py b/scripts/check_store.py new file mode 100644 index 0000000..3bf9081 --- /dev/null +++ b/scripts/check_store.py @@ -0,0 +1,12 @@ +import traceback + +try: + from PluginCore.backend_registry import BackendRegistry + s = BackendRegistry(config={}, suppress_debug=True) + print('INSTANCE TYPE:', type(s)) + print('HAS is_available:', hasattr(s, 'is_available')) + if hasattr(s, 'is_available'): + print('is_available callable:', callable(getattr(s, 'is_available'))) + print('DIR:', sorted([n for n in dir(s) if not n.startswith('__')])) +except Exception: + traceback.print_exc() \ No newline at end of file diff --git a/scripts/check_try_balance.py b/scripts/check_try_balance.py new file mode 100644 index 0000000..c5751ab --- /dev/null +++ b/scripts/check_try_balance.py @@ -0,0 +1,35 @@ +from pathlib import Path +p=Path('SYS/pipeline.py') +s=p.read_text(encoding='utf-8') +lines=s.splitlines() +stack=[] +for i,l in enumerate(lines,1): + stripped=l.strip() + # Skip commented lines + if stripped.startswith('#'): + continue + # compute indent as leading spaces (tabs are converted) + indent = len(l) - len(l.lstrip(' ')) + if stripped.startswith('try:'): + stack.append((indent, i)) + if stripped.startswith('except ') or stripped=='except:' or stripped.startswith('finally:'): + # find the most recent try with same indent + for idx in range(len(stack)-1, -1, -1): + if stack[idx][0] == indent: + stack.pop(idx) + break + else: + # no matching try at same indent + print(f"Found {stripped.split()[0]} at line {i} with no matching try at same indent") + +print('Unmatched try count', len(stack)) +if stack: + print('Unmatched try positions (indent, line):', stack) + for indent, lineno in stack: + start = max(1, lineno - 10) + end = min(len(lines), lineno + 10) + print(f"Context around line {lineno}:") + for i in range(start, end + 1): + print(f"{i:5d}: {lines[i-1]}") +else: + print("All try statements appear matched") diff --git a/scripts/cli_entry.py b/scripts/cli_entry.py index 2c9141b..4629f3f 100644 --- a/scripts/cli_entry.py +++ b/scripts/cli_entry.py @@ -8,7 +8,7 @@ running from a development checkout (by importing the top-level from __future__ import annotations -from typing import Optional, List, Tuple +from typing import Optional, List import importlib import importlib.util import os @@ -64,91 +64,32 @@ def _ensure_repo_root_on_sys_path(pkg_file: Optional[Path] = None) -> Optional[P return None -def _parse_mode_and_strip_args(args: List[str]) -> Tuple[Optional[str], List[str]]: - """Parse --gui/--cli/--mode flags and return (mode, cleaned_args). +def _strip_mode_args(args: List[str]) -> List[str]: + """Strip --gui/--cli/--mode flags from argument list. - The function removes any mode flags from the argument list so the selected - runner can receive the remaining arguments untouched. - - Supported forms: - --gui, -g, --gui=true - --cli, -c, --cli=true - --mode=gui|cli - --mode gui|cli - - Raises ValueError on conflicting or invalid flags. + The GUI/TUI mode has been discontinued. Any mode flags are silently + consumed so they don't interfere with remaining argument parsing. """ - mode: Optional[str] = None out: List[str] = [] i = 0 while i < len(args): a = args[i] la = a.lower() - - # --gui / -g - if la in ("--gui", "-g"): - if mode and mode != "gui": - raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'") - mode = "gui" + if la in ("--gui", "-g", "--cli", "-c"): i += 1 continue - if la.startswith("--gui="): - val = la.split("=", 1)[1] - if val and val not in ("0", "false", "no", "off"): - if mode and mode != "gui": - raise ValueError( - "Conflicting mode flags: found both 'gui' and 'cli'" - ) - mode = "gui" + if la.startswith("--gui=") or la.startswith("--cli="): i += 1 continue - - # --cli / -c - if la in ("--cli", "-c"): - if mode and mode != "cli": - raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'") - mode = "cli" - i += 1 - continue - if la.startswith("--cli="): - val = la.split("=", 1)[1] - if val and val not in ("0", "false", "no", "off"): - if mode and mode != "cli": - raise ValueError( - "Conflicting mode flags: found both 'gui' and 'cli'" - ) - mode = "cli" - i += 1 - continue - - # --mode if la.startswith("--mode="): - val = la.split("=", 1)[1] - val = val.lower() - if val not in ("gui", "cli"): - raise ValueError("--mode must be 'gui' or 'cli'") - if mode and mode != val: - raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'") - mode = val i += 1 continue if la == "--mode": - if i + 1 >= len(args): - raise ValueError("--mode requires a value ('gui' or 'cli')") - val = args[i + 1].lower() - if val not in ("gui", "cli"): - raise ValueError("--mode must be 'gui' or 'cli'") - if mode and mode != val: - raise ValueError("Conflicting mode flags: found both 'gui' and 'cli'") - mode = val i += 2 continue - - # Not a mode flag; keep it out.append(a) i += 1 - - return mode, out + return out def _run_cli(clean_args: List[str]) -> int: @@ -221,15 +162,6 @@ def _run_cli(clean_args: List[str]) -> int: return int(getattr(exc, "code", 0) or 0) -def _run_gui(clean_args: List[str]) -> int: - """Report that the discontinued GUI/TUI mode is no longer available.""" - _ = clean_args - print( - "Error: GUI/TUI mode has been discontinued and is no longer available.", - file=sys.stderr, - ) - return 2 - def main(argv: Optional[List[str]] = None) -> int: """Entry point for console_scripts. @@ -239,11 +171,7 @@ def main(argv: Optional[List[str]] = None) -> int: """ args = list(argv) if argv is not None else list(sys.argv[1:]) - try: - mode, clean_args = _parse_mode_and_strip_args(args) - except ValueError as exc: - print(f"Error parsing mode flags: {exc}", file=sys.stderr) - return 2 + clean_args = _strip_mode_args(args) # Early environment sanity check to detect urllib3/urllib3-future conflicts. # When a broken urllib3 is detected we print an actionable message and @@ -262,10 +190,6 @@ def main(argv: Optional[List[str]] = None) -> int: # startup; we'll continue and let normal import errors surface. pass - # If GUI requested, delegate directly (GUI may decide to honor any args itself) - if mode == "gui": - return _run_gui(clean_args) - # Support quoting a pipeline (or even a single full command) on the command line. # # - If the user provides a single argument that contains a pipe character, diff --git a/scripts/debug_import_vimm.py b/scripts/debug_import_vimm.py new file mode 100644 index 0000000..f954042 --- /dev/null +++ b/scripts/debug_import_vimm.py @@ -0,0 +1,10 @@ +import importlib +import traceback + +try: + m = importlib.import_module('Provider.vimm') + print('Imported', m) + print('Vimm class:', getattr(m, 'Vimm', None)) +except Exception as e: + print('Import failed:', e) + traceback.print_exc() diff --git a/scripts/find_big_indent.py b/scripts/find_big_indent.py new file mode 100644 index 0000000..2c57e66 --- /dev/null +++ b/scripts/find_big_indent.py @@ -0,0 +1,6 @@ +from pathlib import Path +p = Path('Store/registry.py') +for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1): + leading = len(line) - len(line.lstrip(' ')) + if leading > 20: + print(i, leading, repr(line)) \ No newline at end of file diff --git a/scripts/format_tracked.py b/scripts/format_tracked.py new file mode 100644 index 0000000..f242c14 --- /dev/null +++ b/scripts/format_tracked.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Format all tracked Python files using the repo's YAPF style. + +This script intentionally formats only files tracked by git to avoid touching +files that are ignored (e.g., venv, site-packages). +""" +import subprocess +import sys + +try: + out = subprocess.check_output(["git", "ls-files", "*.py"]).decode("utf-8") +except subprocess.CalledProcessError as exc: + print("Failed to get tracked files from git:", exc, file=sys.stderr) + sys.exit(1) + +files = [f for f in (line.strip() for line in out.splitlines()) if f] +print(f"Formatting {len(files)} tracked python files...") +if not files: + print("No tracked python files found.") + sys.exit(0) + +for path in files: + print(path) + subprocess.run([sys.executable, "-m", "yapf", "-i", "--style", ".style.yapf", path], check=False) + +print("Done") diff --git a/scripts/indent_check.py b/scripts/indent_check.py new file mode 100644 index 0000000..03b1c39 --- /dev/null +++ b/scripts/indent_check.py @@ -0,0 +1,11 @@ +import pathlib +p = pathlib.Path('Store/registry.py') +with p.open('r', encoding='utf-8') as f: + lines = f.readlines() +for i in range(312, 328): + if i-1 < len(lines): + line = lines[i-1] + leading = line[:len(line)-len(line.lstrip('\t '))] + print(f"{i}: {repr(line.rstrip())} | leading={repr(leading)} len={len(leading)} chars={[ord(c) for c in leading]}") + else: + print(f"{i}: <EOF>") \ No newline at end of file diff --git a/scripts/indent_stats.py b/scripts/indent_stats.py new file mode 100644 index 0000000..e6c4d58 --- /dev/null +++ b/scripts/indent_stats.py @@ -0,0 +1,10 @@ +from pathlib import Path +p = Path('Store/registry.py') +counts = {} +for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(' ')) + counts[leading] = counts.get(leading, 0) + 1 +for k in sorted(counts.keys()): + print(k, counts[k]) diff --git a/scripts/list_providers.py b/scripts/list_providers.py new file mode 100644 index 0000000..3df26a0 --- /dev/null +++ b/scripts/list_providers.py @@ -0,0 +1,3 @@ +from PluginCore.registry import list_plugins + +print('All plugins:', list_plugins()) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 0c4f92f..f7bf9d8 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -14,6 +14,7 @@ charset-normalizer>=3.2.0 certifi>=2024.12.0 # Optional Telegram support installs telethon>=1.36.0 when [provider=telegram] is configured. internetarchive>=4.1.0 +yt-dlp-ejs # Document and data handling pypdf>=3.0.0 @@ -44,7 +45,8 @@ tqdm>=4.66.0 # Browser automation (for web scraping if needed) playwright>=1.40.0 +paramiko>=3.5.0 +scp>=0.15.0 # Development and utilities python-dateutil>=2.8.0 -