diff --git a/API/HTTP.py b/API/HTTP.py index 5a19857..fda2a89 100644 --- a/API/HTTP.py +++ b/API/HTTP.py @@ -15,7 +15,7 @@ import time import traceback import re import os -from typing import Optional, Dict, Any, Callable, BinaryIO, List, Iterable, Set, Union +from typing import Optional, Dict, Any, Callable, List, Union from pathlib import Path from urllib.parse import unquote, urlparse, parse_qs import logging diff --git a/API/alldebrid.py b/API/alldebrid.py index 2c21f09..b41a71e 100644 --- a/API/alldebrid.py +++ b/API/alldebrid.py @@ -12,7 +12,6 @@ import sys import time from typing import Any, Dict, Optional, Set, List, Sequence, Tuple -import time from urllib.parse import urlparse from SYS.logger import log, debug @@ -1124,7 +1123,7 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any]) # Note: The cmdlet wrapper will handle emitting to pipeline return 0 else: - log(f"❌ Failed to unlock link or already unrestricted", file=sys.stderr) + log("❌ Failed to unlock link or already unrestricted", file=sys.stderr) return 1 diff --git a/API/base.py b/API/base.py index 0567c25..afe972c 100644 --- a/API/base.py +++ b/API/base.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from typing import Any, Dict, Optional from .HTTP import HTTPClient diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index be2b5e5..57188d9 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -92,7 +92,7 @@ "(hitfile\\.net/[a-z0-9A-Z]{4,9})" ], "regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))", - "status": false + "status": true }, "mega": { "name": "mega", @@ -353,7 +353,7 @@ "filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})" ], "regexp": "filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})", - "status": true + "status": false }, "filefactory": { "name": "filefactory", @@ -622,7 +622,7 @@ "(simfileshare\\.net/download/[0-9]+/)" ], "regexp": "(simfileshare\\.net/download/[0-9]+/)", - "status": true + "status": false }, "streamtape": { "name": "streamtape", diff --git a/API/folder.py b/API/folder.py index 87790c8..5030601 100644 --- a/API/folder.py +++ b/API/folder.py @@ -13,8 +13,6 @@ from __future__ import annotations import sqlite3 import json import logging -import subprocess -import shutil import time import os from contextlib import contextmanager @@ -303,8 +301,21 @@ class API_folder_store: if should_check_empty: # Check if there are any files or directories in the library root (excluding the DB itself if it was just created) - # We use a generator and next() for efficiency. existing_items = [item for item in self.library_root.iterdir() if item.name != self.DB_NAME] + + # Allow an empty 'incoming' directory created by upload flow to exist + # (this prevents a false-positive safety check when an upload endpoint + # creates the incoming dir before DB initialization). + if existing_items: + if len(existing_items) == 1 and existing_items[0].name == "incoming" and existing_items[0].is_dir(): + try: + # If the incoming directory is empty, treat it as harmless. + if not any(existing_items[0].iterdir()): + existing_items = [] + except Exception: + # If we can't inspect it safely, leave the original items in place + pass + if existing_items: # Log the items found for debugging item_names = [i.name for i in existing_items[:5]] @@ -1378,7 +1389,7 @@ class API_folder_store: (file_hash, existing_title[0]), ) - logger.debug(f"[save_tags] Preserved existing title tag") + logger.debug("[save_tags] Preserved existing title tag") elif not existing_title and not new_title_provided: filename_without_ext = abs_path.stem if filename_without_ext: diff --git a/API/loc.py b/API/loc.py index bd6effe..9a71d93 100644 --- a/API/loc.py +++ b/API/loc.py @@ -12,7 +12,6 @@ The LoC JSON API does not require an API key. from __future__ import annotations -import json from typing import Any, Dict, Optional from .base import API, ApiError diff --git a/API/podcastindex.py b/API/podcastindex.py index 7a642a5..47900ef 100644 --- a/API/podcastindex.py +++ b/API/podcastindex.py @@ -12,7 +12,6 @@ Authentication headers required for most endpoints: from __future__ import annotations import hashlib -import json import time from typing import Any, Dict, List, Optional diff --git a/API/zerotier.py b/API/zerotier.py index 9505bec..c33998c 100644 --- a/API/zerotier.py +++ b/API/zerotier.py @@ -32,7 +32,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from SYS.logger import debug, log +from SYS.logger import debug # Optional Python ZeroTier bindings - prefer them when available _HAVE_PY_ZEROTIER = False diff --git a/CLI.py b/CLI.py index 8ea9130..7cd58f0 100644 --- a/CLI.py +++ b/CLI.py @@ -7,8 +7,6 @@ This module intentionally uses a class-based architecture: - all REPL/pipeline/cmdlet execution state lives on objects """ -import atexit -import io import json import re import shlex @@ -17,15 +15,14 @@ import threading import time import uuid from copy import deepcopy -from datetime import datetime + from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, TextIO, Tuple, cast +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, cast import typer from prompt_toolkit import PromptSession from prompt_toolkit.completion import Completer, Completion from prompt_toolkit.document import Document -from prompt_toolkit.lexers import Lexer from prompt_toolkit.styles import Style from rich.console import Console from rich.layout import Layout @@ -34,9 +31,6 @@ from rich.markdown import Markdown from rich.bar import Bar from rich.table import Table as RichTable from SYS.rich_display import ( - IMAGE_EXTENSIONS, - render_image_to_console, - render_item_details_panel, stderr_console, stdout_console, ) @@ -59,7 +53,6 @@ def _install_rich_traceback(*, show_locals: bool = False) -> None: # Default to Rich tracebacks for the whole process. _install_rich_traceback(show_locals=False) -from SYS.background_notifier import ensure_background_notifier from SYS.logger import debug, set_debug from SYS.worker_manager import WorkerManager from SYS.background_services import ensure_zerotier_server_running, stop_zerotier_server @@ -72,32 +65,15 @@ from SYS.cmdlet_catalog import ( list_cmdlet_metadata, list_cmdlet_names, ) -from SYS.config import get_local_storage_path, load_config +from SYS.config import load_config from SYS.result_table import Table + +from SYS.worker import WorkerManagerRegistry, WorkerStages, WorkerOutputMirror, WorkerStageSession +from SYS.pipeline import PipelineExecutor from ProviderCore.registry import provider_inline_query_choices -HELP_EXAMPLE_SOURCE_COMMANDS = { - ".help-example", - "help-example", -} -def _split_pipeline_tokens(tokens: Sequence[str]) -> List[List[str]]: - """Split example tokens into per-stage command sequences using pipe separators.""" - - stages: List[List[str]] = [] - current: List[str] = [] - for token in tokens: - if token == "|": - if current: - stages.append(current) - current = [] - continue - current.append(str(token)) - if current: - stages.append(current) - return [stage for stage in stages if stage] - # Selection parsing and REPL lexer moved to SYS.cli_parsing from SYS.cli_parsing import SelectionSyntax, SelectionFilterSyntax, MedeiaLexer @@ -106,222 +82,15 @@ from SYS.cli_parsing import SelectionSyntax, SelectionFilterSyntax, MedeiaLexer -class WorkerOutputMirror(io.TextIOBase): - """Mirror stdout/stderr to worker manager while preserving console output.""" - - def __init__( - self, - original: TextIO, - manager: WorkerManager, - worker_id: str, - channel: str - ): - self._original = original - self._manager = manager - self._worker_id = worker_id - self._channel = channel - self._pending: str = "" - - def write(self, data: str) -> int: # type: ignore[override] - if not data: - return 0 - self._original.write(data) - self._buffer_text(data) - return len(data) - - def flush(self) -> None: # type: ignore[override] - self._original.flush() - self._flush_pending(force=True) - - def isatty(self) -> bool: # pragma: no cover - return bool(getattr(self._original, "isatty", lambda: False)()) - - def _buffer_text(self, data: str) -> None: - combined = self._pending + data - lines = combined.splitlines(keepends=True) - if not lines: - self._pending = combined - return - - if lines[-1].endswith(("\n", "\r")): - complete = lines - self._pending = "" - else: - complete = lines[:-1] - self._pending = lines[-1] - - for chunk in complete: - self._emit(chunk) - - def _flush_pending(self, *, force: bool = False) -> None: - if self._pending and force: - self._emit(self._pending) - self._pending = "" - - def _emit(self, text: str) -> None: - if not text: - return - try: - self._manager.append_stdout(self._worker_id, text, channel=self._channel) - except Exception: - pass - - @property - def encoding(self) -> str: # type: ignore[override] - return getattr(self._original, "encoding", "utf-8") -class WorkerStageSession: - """Lifecycle helper for wrapping a CLI cmdlet execution in a worker record.""" - - def __init__( - self, - *, - manager: WorkerManager, - worker_id: str, - orig_stdout: TextIO, - orig_stderr: TextIO, - stdout_proxy: WorkerOutputMirror, - stderr_proxy: WorkerOutputMirror, - config: Optional[Dict[str, - Any]], - logging_enabled: bool, - completion_label: str, - error_label: str, - ) -> None: - self.manager = manager - self.worker_id = worker_id - self.orig_stdout = orig_stdout - self.orig_stderr = orig_stderr - self.stdout_proxy = stdout_proxy - self.stderr_proxy = stderr_proxy - self.config = config - self.logging_enabled = logging_enabled - self.closed = False - self._completion_label = completion_label - self._error_label = error_label - - def close(self, *, status: str = "completed", error_msg: str = "") -> None: - if self.closed: - return - try: - self.stdout_proxy.flush() - self.stderr_proxy.flush() - except Exception: - pass - - sys.stdout = self.orig_stdout - sys.stderr = self.orig_stderr - - if self.logging_enabled: - try: - self.manager.disable_logging_for_worker(self.worker_id) - except Exception: - pass - - try: - if status == "completed": - self.manager.log_step(self.worker_id, self._completion_label) - else: - self.manager.log_step( - self.worker_id, - f"{self._error_label}: {error_msg or status}" - ) - except Exception: - pass - - try: - self.manager.finish_worker( - self.worker_id, - result=status or "completed", - error_msg=error_msg or "" - ) - except Exception: - pass - - if self.config and self.config.get("_current_worker_id") == self.worker_id: - self.config.pop("_current_worker_id", None) - self.closed = True -class WorkerManagerRegistry: - """Process-wide WorkerManager cache keyed by library_root.""" - - _manager: Optional[WorkerManager] = None - _manager_root: Optional[Path] = None - _orphan_cleanup_done: bool = False - _registered: bool = False - - @classmethod - def ensure(cls, config: Dict[str, Any]) -> Optional[WorkerManager]: - if not isinstance(config, dict): - return None - - existing = config.get("_worker_manager") - if isinstance(existing, WorkerManager): - return existing - - library_root = get_local_storage_path(config) - if not library_root: - return None - - try: - resolved_root = Path(library_root).resolve() - except Exception: - resolved_root = Path(library_root) - - try: - if cls._manager is None or cls._manager_root != resolved_root: - if cls._manager is not None: - try: - cls._manager.close() - except Exception: - pass - cls._manager = WorkerManager(resolved_root, auto_refresh_interval=0.5) - cls._manager_root = resolved_root - - manager = cls._manager - config["_worker_manager"] = manager - - if manager is not None and not cls._orphan_cleanup_done: - try: - manager.expire_running_workers( - older_than_seconds=120, - worker_id_prefix="cli_%", - reason= - "CLI session ended unexpectedly; marking worker as failed", - ) - except Exception: - pass - else: - cls._orphan_cleanup_done = True - - if not cls._registered: - atexit.register(cls.close) - cls._registered = True - - return manager - except Exception as exc: - print( - f"[worker] Could not initialize worker manager: {exc}", - file=sys.stderr - ) - return None - - @classmethod - def close(cls) -> None: - if cls._manager is None: - return - try: - cls._manager.close() - except Exception: - pass - cls._manager = None - cls._manager_root = None - cls._orphan_cleanup_done = False -class WorkerStages: + + +class _OldWorkerStages: """Factory methods for stage/pipeline worker sessions.""" @staticmethod @@ -772,11 +541,8 @@ class CmdletCompleter(Completer): yield Completion("-help", start_position=-len(current_token)) -class MedeiaLexer(Lexer): +# Lexer implementation removed; use `MedeiaLexer` from `SYS.cli_parsing` instead. - def lex_document(self, document: Document): # type: ignore[override] - - def get_line(lineno: int): line = document.lines[lineno] tokens: List[tuple[str, str]] = [] @@ -886,8 +652,6 @@ class MedeiaLexer(Lexer): return get_line -from SYS.cli_parsing import MedeiaLexer as _MigratedMedeiaLexer -MedeiaLexer = _MigratedMedeiaLexer class ConfigLoader: @@ -1531,11 +1295,11 @@ class CmdletExecutor: if cmd_name in self_managing_commands: table = ctx.get_last_result_table() if table is None: - table = ResultTable(table_title) + table = Table(table_title) for emitted in emits: table.add_result(emitted) else: - table = ResultTable(table_title) + table = Table(table_title) for emitted in emits: table.add_result(emitted) @@ -1668,2384 +1432,6 @@ class CmdletExecutor: stage_session.close(status=stage_status, error_msg=stage_error) -class PipelineExecutor: - - def __init__(self, *, config_loader: ConfigLoader) -> None: - self._config_loader = config_loader - self._toolbar_output: Optional[Callable[[str], None]] = None - - def set_toolbar_output(self, output: Optional[Callable[[str], None]]) -> None: - self._toolbar_output = output - - @staticmethod - def _split_stages(tokens: Sequence[str]) -> List[List[str]]: - stages: List[List[str]] = [] - current: List[str] = [] - for token in tokens: - if token == "|": - if current: - stages.append(current) - current = [] - else: - current.append(token) - if current: - stages.append(current) - return stages - - @staticmethod - def _validate_download_file_relationship_order(stages: List[List[str]]) -> bool: - """Guard against running add-relationship on unstored download-file results. - - Intended UX: - download-file ... | add-file -store | add-relationship - - Rationale: - download-file outputs items that may not yet have a stable store+hash. - add-relationship is designed to operate in store/hash mode. - """ - - def _norm(name: str) -> str: - return str(name or "").replace("_", "-").strip().lower() - - names: List[str] = [] - for stage in stages or []: - if not stage: - continue - names.append(_norm(stage[0])) - - dl_idxs = [i for i, n in enumerate(names) if n == "download-file"] - rel_idxs = [i for i, n in enumerate(names) if n == "add-relationship"] - add_file_idxs = [i for i, n in enumerate(names) if n == "add-file"] - - if not dl_idxs or not rel_idxs: - return True - - # If download-file is upstream of add-relationship, require an add-file in between. - for rel_i in rel_idxs: - dl_before = [d for d in dl_idxs if d < rel_i] - if not dl_before: - continue - dl_i = max(dl_before) - if not any(dl_i < a < rel_i for a in add_file_idxs): - print( - "Pipeline order error: when using download-file with add-relationship, " - "add-relationship must come after add-file (so items are stored and have store+hash).\n" - "Example: download-file <...> | add-file -store | add-relationship\n" - ) - return False - - return True - - @staticmethod - def _try_clear_pipeline_stop(ctx: Any) -> None: - try: - if hasattr(ctx, "clear_pipeline_stop"): - ctx.clear_pipeline_stop() - except Exception: - pass - - @staticmethod - def _maybe_seed_current_stage_table(ctx: Any) -> None: - try: - if hasattr(ctx, - "get_current_stage_table") and not ctx.get_current_stage_table(): - display_table = ( - ctx.get_display_table() if hasattr(ctx, - "get_display_table") else None - ) - if display_table: - ctx.set_current_stage_table(display_table) - else: - last_table = ( - ctx.get_last_result_table() - if hasattr(ctx, - "get_last_result_table") else None - ) - if last_table: - ctx.set_current_stage_table(last_table) - except Exception: - pass - - @staticmethod - def _maybe_apply_pending_pipeline_tail(ctx: Any, - stages: List[List[str]]) -> List[List[str]]: - try: - pending_tail = ( - ctx.get_pending_pipeline_tail() - if hasattr(ctx, - "get_pending_pipeline_tail") else [] - ) - pending_source = ( - ctx.get_pending_pipeline_source() - if hasattr(ctx, - "get_pending_pipeline_source") else None - ) - except Exception: - pending_tail = [] - pending_source = None - - try: - current_source = ( - ctx.get_current_stage_table_source_command() - if hasattr(ctx, - "get_current_stage_table_source_command") else None - ) - except Exception: - current_source = None - - try: - effective_source = current_source or ( - ctx.get_last_result_table_source_command() - if hasattr(ctx, - "get_last_result_table_source_command") else None - ) - except Exception: - effective_source = current_source - - selection_start = bool( - stages and stages[0] and stages[0][0].startswith("@") - ) - - def _tail_is_suffix(existing: List[List[str]], tail: List[List[str]]) -> bool: - if not tail or not existing: - return False - if len(tail) > len(existing): - return False - return existing[-len(tail):] == tail - - if pending_tail and selection_start: - if (pending_source is None) or (effective_source - and pending_source == effective_source): - # Only append the pending tail if the user hasn't already provided it. - if not _tail_is_suffix(stages, pending_tail): - stages = list(stages) + list(pending_tail) - try: - if hasattr(ctx, "clear_pending_pipeline_tail"): - ctx.clear_pending_pipeline_tail() - except Exception: - pass - else: - try: - if hasattr(ctx, "clear_pending_pipeline_tail"): - ctx.clear_pending_pipeline_tail() - except Exception: - pass - return stages - - def _apply_quiet_background_flag(self, config: Any) -> Any: - if isinstance(config, dict): - # This executor is used by both the REPL and the `pipeline` subcommand. - # Quiet/background mode is helpful for detached/background runners, but - # it suppresses interactive UX (like the pipeline Live progress UI). - config["_quiet_background_output"] = bool(self._toolbar_output is None) - return config - - @staticmethod - def _extract_first_stage_selection_tokens( - stages: List[List[str]], - ) -> tuple[List[List[str]], - List[int], - bool, - bool]: - first_stage_tokens = stages[0] if stages else [] - first_stage_selection_indices: List[int] = [] - first_stage_had_extra_args = False - first_stage_select_all = False - - if first_stage_tokens: - new_first_stage: List[str] = [] - for token in first_stage_tokens: - if token.startswith("@"): # selection - selection = SelectionSyntax.parse(token) - if selection is not None: - first_stage_selection_indices = sorted( - [i - 1 for i in selection] - ) - continue - if token == "@*": - first_stage_select_all = True - continue - new_first_stage.append(token) - - if new_first_stage: - stages = list(stages) - stages[0] = new_first_stage - if first_stage_selection_indices or first_stage_select_all: - first_stage_had_extra_args = True - elif first_stage_selection_indices or first_stage_select_all: - stages = list(stages) - stages.pop(0) - - return ( - stages, - first_stage_selection_indices, - first_stage_had_extra_args, - first_stage_select_all, - ) - - @staticmethod - def _apply_select_all_if_requested(ctx: Any, - indices: List[int], - select_all: bool) -> List[int]: - if not select_all: - return indices - try: - last_items = ctx.get_last_result_items() - except Exception: - last_items = None - if last_items: - return list(range(len(last_items))) - return indices - - @staticmethod - def _maybe_run_class_selector( - ctx: Any, - config: Any, - selected_items: list, - *, - stage_is_last: bool - ) -> bool: - if not stage_is_last: - return False - - candidates: list[str] = [] - seen: set[str] = set() - - def _add(value) -> None: - try: - text = str(value or "").strip().lower() - except Exception: - return - if not text or text in seen: - return - seen.add(text) - candidates.append(text) - - try: - current_table = ctx.get_current_stage_table() or ctx.get_last_result_table() - _add( - current_table. - table if current_table and hasattr(current_table, - "table") else None - ) - - # Prefer an explicit provider hint from table metadata when available. - # This keeps @N selectors working even when row payloads don't carry a - # provider key (or when they carry a table-type like tidal.album). - try: - meta = ( - current_table.get_table_metadata() - if current_table is not None and hasattr(current_table, "get_table_metadata") - else getattr(current_table, "table_metadata", None) - ) - except Exception: - meta = None - if isinstance(meta, dict): - _add(meta.get("provider")) - except Exception: - pass - - for item in selected_items or []: - if isinstance(item, dict): - _add(item.get("provider")) - _add(item.get("store")) - _add(item.get("table")) - else: - _add(getattr(item, "provider", None)) - _add(getattr(item, "store", None)) - _add(getattr(item, "table", None)) - - try: - from ProviderCore.registry import get_provider, is_known_provider_name - except Exception: - get_provider = None # type: ignore - is_known_provider_name = None # type: ignore - - # If we have a table-type like "tidal.album", also try its provider prefix ("tidal") - # when that prefix is a registered provider name. - if is_known_provider_name is not None: - try: - for key in list(candidates): - if not isinstance(key, str): - continue - if "." not in key: - continue - if is_known_provider_name(key): - continue - prefix = str(key).split(".", 1)[0].strip().lower() - if prefix and is_known_provider_name(prefix): - _add(prefix) - except Exception: - pass - - if get_provider is not None: - for key in candidates: - try: - if is_known_provider_name is not None and ( - not is_known_provider_name(key)): - continue - except Exception: - # If the predicate fails for any reason, fall back to legacy behavior. - pass - try: - provider = get_provider(key, config) - except Exception: - continue - selector = getattr(provider, "selector", None) - if selector is None: - continue - try: - handled = bool( - selector(selected_items, - ctx=ctx, - stage_is_last=True) - ) - except Exception as exc: - print(f"{key} selector failed: {exc}\n") - return True - if handled: - return True - - store_keys: list[str] = [] - for item in selected_items or []: - if isinstance(item, dict): - v = item.get("store") - else: - v = getattr(item, "store", None) - name = str(v or "").strip() - if name: - store_keys.append(name) - - if store_keys: - try: - from Store.registry import Store as StoreRegistry - - store_registry = StoreRegistry(config, suppress_debug=True) - _backend_names = list(store_registry.list_backends() or []) - _backend_by_lower = { - str(n).lower(): str(n) - for n in _backend_names if str(n).strip() - } - for name in store_keys: - resolved_name = name - if not store_registry.is_available(resolved_name): - resolved_name = _backend_by_lower.get(str(name).lower(), name) - if not store_registry.is_available(resolved_name): - continue - backend = store_registry[resolved_name] - selector = getattr(backend, "selector", None) - if selector is None: - continue - handled = bool( - selector(selected_items, - ctx=ctx, - stage_is_last=True) - ) - if handled: - return True - except Exception: - pass - - return False - - @staticmethod - def _summarize_stage_text(stage_tokens: Sequence[str], limit: int = 140) -> str: - combined = " ".join(str(tok) for tok in stage_tokens if tok is not None).strip() - if not combined: - return "" - normalized = re.sub(r"\s+", " ", combined) - if len(normalized) <= limit: - return normalized - return normalized[:limit - 3].rstrip() + "..." - - @staticmethod - def _log_pipeline_event( - worker_manager: Any, - worker_id: Optional[str], - message: str, - ) -> None: - if not worker_manager or not worker_id or not message: - return - try: - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - except Exception: - timestamp = "" - if timestamp: - text = f"{timestamp} - PIPELINE - {message}" - else: - text = f"PIPELINE - {message}" - try: - worker_manager.append_stdout(worker_id, text + "\n", channel="log") - except Exception: - pass - - @staticmethod - def _maybe_open_url_selection( - current_table: Any, - selected_items: list, - *, - stage_is_last: bool - ) -> bool: - if not stage_is_last: - return False - if not selected_items or len(selected_items) != 1: - return False - - table_type = "" - source_cmd = "" - try: - table_type = str(getattr(current_table, "table", "") or "").strip().lower() - except Exception: - table_type = "" - try: - source_cmd = ( - str(getattr(current_table, - "source_command", - "") or "").strip().replace("_", - "-").lower() - ) - except Exception: - source_cmd = "" - - if table_type != "url" and source_cmd != "get-url": - return False - - item = selected_items[0] - url = None - try: - from cmdlet._shared import get_field - - url = get_field(item, "url") - except Exception: - try: - url = item.get("url") if isinstance(item, - dict - ) else getattr(item, - "url", - None) - except Exception: - url = None - - url_text = str(url or "").strip() - if not url_text: - return False - - try: - import webbrowser - - webbrowser.open(url_text, new=2) - return True - except Exception: - return False - - def _maybe_enable_background_notifier( - self, - worker_manager: Any, - config: Any, - pipeline_session: Any - ) -> None: - if not (pipeline_session and worker_manager and isinstance(config, dict)): - return - - session_worker_ids = config.get("_session_worker_ids") - if not session_worker_ids: - return - - try: - output_fn = self._toolbar_output - quiet_mode = bool(config.get("_quiet_background_output")) - terminal_only = quiet_mode and not output_fn - kwargs: Dict[str, - Any] = { - "session_worker_ids": session_worker_ids, - "only_terminal_updates": terminal_only, - "overlay_mode": bool(output_fn), - } - if output_fn: - kwargs["output"] = output_fn - ensure_background_notifier(worker_manager, **kwargs) - except Exception: - pass - - @staticmethod - def _get_raw_stage_texts(ctx: Any) -> List[str]: - raw_stage_texts: List[str] = [] - try: - if hasattr(ctx, "get_current_command_stages"): - raw_stage_texts = ctx.get_current_command_stages() or [] - except Exception: - raw_stage_texts = [] - return raw_stage_texts - - def _maybe_apply_initial_selection( - self, - ctx: Any, - config: Any, - stages: List[List[str]], - *, - selection_indices: List[int], - first_stage_had_extra_args: bool, - worker_manager: Any, - pipeline_session: Any, - ) -> tuple[bool, - Any]: - if not selection_indices: - return True, None - - # Selection should operate on the *currently displayed* selectable table. - # Some navigation flows (e.g. @.. back) can show a display table without - # updating current_stage_table. Provider selectors rely on current_stage_table - # to detect table type (e.g. tidal.album -> tracks), so sync it here. - display_table = None - try: - display_table = ( - ctx.get_display_table() if hasattr(ctx, "get_display_table") else None - ) - except Exception: - display_table = None - - current_stage_table = None - try: - current_stage_table = ( - ctx.get_current_stage_table() - if hasattr(ctx, "get_current_stage_table") else None - ) - except Exception: - current_stage_table = None - - try: - if display_table is not None and hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(display_table) - elif current_stage_table is None and hasattr(ctx, "set_current_stage_table"): - last_table = ( - ctx.get_last_result_table() - if hasattr(ctx, "get_last_result_table") else None - ) - if last_table is not None: - ctx.set_current_stage_table(last_table) - except Exception: - pass - - source_cmd = None - source_args_raw = None - try: - source_cmd = ctx.get_current_stage_table_source_command() - source_args_raw = ctx.get_current_stage_table_source_args() - except Exception: - source_cmd = None - source_args_raw = None - - if isinstance(source_args_raw, str): - source_args: List[str] = [source_args_raw] - elif isinstance(source_args_raw, list): - source_args = [str(x) for x in source_args_raw if x is not None] - else: - source_args = [] - - current_table = None - try: - current_table = ctx.get_current_stage_table() - except Exception: - current_table = None - table_type = ( - current_table.table if current_table and hasattr(current_table, - "table") else None - ) - - command_expanded = False - example_selector_triggered = False - normalized_source_cmd = str(source_cmd or "").replace("_", "-").strip().lower() - - if normalized_source_cmd in HELP_EXAMPLE_SOURCE_COMMANDS and selection_indices: - try: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - except Exception: - row_args = None - tokens: List[str] = [] - if isinstance(row_args, list) and row_args: - tokens = [str(x) for x in row_args if x is not None] - if tokens: - stage_groups = _split_pipeline_tokens(tokens) - if stage_groups: - for stage in reversed(stage_groups): - stages.insert(0, stage) - selection_indices = [] - command_expanded = True - example_selector_triggered = True - - if not example_selector_triggered: - if table_type in {"youtube", - "soulseek"}: - command_expanded = False - elif source_cmd == "search-file" and source_args and "youtube" in source_args: - command_expanded = False - else: - selected_row_args: List[str] = [] - skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0 - # Command expansion via @N: - # - Default behavior: expand ONLY for single-row selections. - # - Special case: allow multi-row expansion for add-file directory tables by - # combining selected rows into a single `-path file1,file2,...` argument. - if source_cmd and not skip_pipe_expansion: - src = str(source_cmd).replace("_", "-").strip().lower() - - if src == "add-file" and selection_indices: - row_args_list: List[List[str]] = [] - for idx in selection_indices: - try: - row_args = ctx.get_current_stage_table_row_selection_args( - idx - ) - except Exception: - row_args = None - if isinstance(row_args, list) and row_args: - row_args_list.append( - [str(x) for x in row_args if x is not None] - ) - - # Combine `['-path', ]` from each row into one `-path` arg. - paths: List[str] = [] - can_merge = bool(row_args_list) and ( - len(row_args_list) == len(selection_indices) - ) - if can_merge: - for ra in row_args_list: - if len(ra) == 2 and str(ra[0]).strip().lower() in { - "-path", - "--path", - "-p", - }: - p = str(ra[1]).strip() - if p: - paths.append(p) - else: - can_merge = False - break - - if can_merge and paths: - selected_row_args.extend(["-path", ",".join(paths)]) - elif len(selection_indices) == 1 and row_args_list: - selected_row_args.extend(row_args_list[0]) - else: - # Only perform @N command expansion for *single-item* selections. - # For multi-item selections (e.g. @*, @1-5), expanding to one row - # would silently drop items. In those cases we pipe items downstream. - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - if row_args: - selected_row_args.extend(row_args) - - if selected_row_args: - if isinstance(source_cmd, list): - cmd_list: List[str] = [str(x) for x in source_cmd if x is not None] - elif isinstance(source_cmd, str): - cmd_list = [source_cmd] - else: - cmd_list = [] - - # IMPORTANT: Put selected row args *before* source_args. - # Rationale: The cmdlet argument parser treats the *first* unknown - # token as a positional value (e.g., URL). If `source_args` - # contain unknown flags (like -provider which download-file does - # not declare), they could be misinterpreted as the positional - # URL argument and cause attempts to download strings like - # "-provider" (which is invalid). By placing selection args - # first we ensure the intended URL/selection token is parsed - # as the positional URL and avoid this class of parsing errors. - expanded_stage: List[str] = cmd_list + selected_row_args + source_args - - if first_stage_had_extra_args and stages: - expanded_stage += stages[0] - stages[0] = expanded_stage - else: - stages.insert(0, expanded_stage) - - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", - ) - except Exception: - pass - - stage_table = None - try: - stage_table = ctx.get_current_stage_table() - except Exception: - stage_table = None - - display_table = None - try: - display_table = ( - ctx.get_display_table() if hasattr(ctx, - "get_display_table") else None - ) - except Exception: - display_table = None - - if not stage_table and display_table is not None: - stage_table = display_table - if not stage_table: - try: - stage_table = ctx.get_last_result_table() - except Exception: - stage_table = None - - # Prefer selecting from the last selectable *table* (search/playlist) - # rather than from display-only emitted items, unless we're explicitly - # selecting from an overlay table. - try: - if display_table is not None and stage_table is display_table: - items_list = ctx.get_last_result_items() or [] - else: - if hasattr(ctx, "get_last_selectable_result_items"): - items_list = ctx.get_last_selectable_result_items() or [] - else: - items_list = ctx.get_last_result_items() or [] - except Exception: - items_list = [] - - resolved_items = items_list if items_list else [] - if items_list: - filtered = [ - resolved_items[i] for i in selection_indices - if 0 <= i < len(resolved_items) - ] - if not filtered: - print("No items matched selection in pipeline\n") - return False, None - - # Provider selection expansion (non-terminal): allow certain provider tables - # (e.g. tidal.album) to expand to multiple downstream items when the user - # pipes into another stage (e.g. @N | .mpv or @N | add-file). - table_type_hint = None - try: - table_type_hint = ( - stage_table.table - if stage_table is not None and hasattr(stage_table, "table") - else None - ) - except Exception: - table_type_hint = None - - if stages and isinstance(table_type_hint, str) and table_type_hint.strip().lower() == "tidal.album": - try: - from ProviderCore.registry import get_provider - - prov = get_provider("tidal", config) - except Exception: - prov = None - - if prov is not None and hasattr(prov, "_extract_album_selection_context") and hasattr(prov, "_tracks_for_album"): - try: - album_contexts = prov._extract_album_selection_context(filtered) # type: ignore[attr-defined] - except Exception: - album_contexts = [] - - track_items: List[Any] = [] - seen_track_ids: set[int] = set() - for album_id, album_title, artist_name in album_contexts or []: - try: - track_results = prov._tracks_for_album( # type: ignore[attr-defined] - album_id=album_id, - album_title=album_title, - artist_name=artist_name, - limit=500, - ) - except Exception: - track_results = [] - for tr in track_results or []: - try: - md = getattr(tr, "full_metadata", None) - tid = None - if isinstance(md, dict): - raw_id = md.get("trackId") or md.get("id") - try: - tid = int(raw_id) if raw_id is not None else None - except Exception: - tid = None - if tid is not None: - if tid in seen_track_ids: - continue - seen_track_ids.add(tid) - except Exception: - pass - track_items.append(tr) - - if track_items: - filtered = track_items - table_type_hint = "tidal.track" - - if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(not stages)): - return False, None - - from cmdlet._shared import coerce_to_pipe_object - - filtered_pipe_objs = [coerce_to_pipe_object(item) for item in filtered] - piped_result = ( - filtered_pipe_objs - if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] - ) - - if pipeline_session and worker_manager: - try: - selection_parts = [f"@{i+1}" for i in selection_indices] - worker_manager.log_step( - pipeline_session.worker_id, - f"Applied @N selection {' | '.join(selection_parts)}", - ) - except Exception: - pass - - # Auto-insert downloader stages for provider tables. - try: - current_table = ctx.get_current_stage_table() - if current_table is None and hasattr(ctx, "get_display_table"): - current_table = ctx.get_display_table() - if current_table is None: - current_table = ctx.get_last_result_table() - except Exception: - current_table = None - table_type = None - try: - if isinstance(table_type_hint, str) and table_type_hint.strip(): - table_type = table_type_hint - else: - table_type = ( - current_table.table - if current_table and hasattr(current_table, "table") else None - ) - except Exception: - table_type = ( - current_table.table - if current_table and hasattr(current_table, "table") else None - ) - - def _norm_cmd(name: Any) -> str: - return str(name or "").replace("_", "-").strip().lower() - - auto_stage = None - if isinstance(table_type, str) and table_type: - try: - from ProviderCore.registry import selection_auto_stage_for_table - - auto_stage = selection_auto_stage_for_table(table_type) - except Exception: - auto_stage = None - - source_cmd_for_selection = None - source_args_for_selection: List[str] = [] - try: - source_cmd_for_selection = ( - ctx.get_current_stage_table_source_command() - or ctx.get_last_result_table_source_command() - ) - source_args_for_selection = ( - ctx.get_current_stage_table_source_args() - or ctx.get_last_result_table_source_args() - or [] - ) - except Exception: - source_cmd_for_selection = None - source_args_for_selection = [] - - if not stages and selection_indices and source_cmd_for_selection: - src_norm = _norm_cmd(source_cmd_for_selection) - if src_norm in {".worker", "worker", "workers"}: - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = None - try: - row_args = ctx.get_current_stage_table_row_selection_args(idx) - except Exception: - row_args = None - if not row_args: - try: - row_args = ctx.get_last_result_table_row_selection_args(idx) - except Exception: - row_args = None - if not row_args: - try: - items = ctx.get_last_result_items() or [] - if 0 <= idx < len(items): - maybe = items[idx] - if isinstance(maybe, dict): - candidate = maybe.get("_selection_args") - if isinstance(candidate, (list, tuple)): - row_args = [str(x) for x in candidate if x is not None] - except Exception: - row_args = row_args or None - - if row_args: - stages.append( - [str(source_cmd_for_selection)] - + [str(x) for x in row_args if x is not None] - + [str(x) for x in source_args_for_selection if x is not None] - ) - - def _apply_row_action_to_stage(stage_idx: int) -> bool: - if not selection_indices or len(selection_indices) != 1: - return False - try: - row_action = ctx.get_current_stage_table_row_selection_action( - selection_indices[0] - ) - except Exception: - row_action = None - if not row_action: - # Fallback to serialized payload when the table row is unavailable - try: - items = ctx.get_last_result_items() or [] - if 0 <= selection_indices[0] < len(items): - maybe = items[selection_indices[0]] - if isinstance(maybe, dict): - candidate = maybe.get("_selection_action") - if isinstance(candidate, (list, tuple)): - row_action = [str(x) for x in candidate if x is not None] - debug(f"@N row {selection_indices[0]} restored action from payload: {row_action}") - except Exception: - row_action = row_action or None - if not row_action: - debug(f"@N row {selection_indices[0]} has no selection_action") - return False - normalized = [str(x) for x in row_action if x is not None] - if not normalized: - return False - debug(f"Applying row action for row {selection_indices[0]} -> {normalized}") - if 0 <= stage_idx < len(stages): - debug(f"Replacing stage {stage_idx} {stages[stage_idx]} with row action {normalized}") - stages[stage_idx] = normalized - return True - return False - - if not stages: - if isinstance(table_type, str) and table_type.startswith("metadata."): - print("Auto-applying metadata selection via get-tag") - stages.append(["get-tag"]) - elif auto_stage: - try: - print(f"Auto-running selection via {auto_stage[0]}") - except Exception: - pass - # Append the auto stage now. If the user also provided a selection - # (e.g., @1 | add-file ...), we want to attach the row selection - # args *to the auto-inserted stage* so the download command receives - # the selected row information immediately. - stages.append(list(auto_stage)) - debug(f"Inserted auto stage before row action: {stages[-1]}") - - # If the caller included a selection (e.g., @1) try to attach - # the selection args immediately to the inserted auto stage so - # the expansion is effective in a single pass. - if selection_indices: - try: - if not _apply_row_action_to_stage(len(stages) - 1): - # Only support single-row selection for auto-attach here - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - if not row_args: - try: - items = ctx.get_last_result_items() or [] - if 0 <= idx < len(items): - maybe = items[idx] - if isinstance(maybe, dict): - candidate = maybe.get("_selection_args") - if isinstance(candidate, (list, tuple)): - row_args = [str(x) for x in candidate if x is not None] - except Exception: - row_args = row_args or None - if row_args: - # Place selection args before any existing source args - inserted = stages[-1] - if inserted: - cmd = inserted[0] - tail = [str(x) for x in inserted[1:]] - stages[-1] = [cmd] + [str(x) for x in row_args] + tail - except Exception: - pass - else: - first_cmd = stages[0][0] if stages and stages[0] else None - if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in ( - "get-tag", - "get_tag", - ".pipe", - ".mpv", - ): - print("Auto-inserting get-tag after metadata selection") - stages.insert(0, ["get-tag"]) - elif auto_stage: - first_cmd_norm = _norm_cmd(first_cmd) - auto_cmd_norm = _norm_cmd(auto_stage[0]) - if first_cmd_norm not in (auto_cmd_norm, ".pipe", ".mpv"): - debug(f"Auto-inserting {auto_cmd_norm} after selection") - # Insert the auto stage before the user-specified stage - # Note: Do NOT append source_args here - they are search tokens from - # the previous stage and should not be passed to the downloader. - stages.insert(0, list(auto_stage)) - debug(f"Inserted auto stage before existing pipeline: {stages[0]}") - - # If a selection is present, attach the row selection args to the - # newly-inserted stage so the download stage runs with the - # selected row information. - if selection_indices: - try: - if not _apply_row_action_to_stage(0): - if len(selection_indices) == 1: - idx = selection_indices[0] - row_args = ctx.get_current_stage_table_row_selection_args(idx) - if not row_args: - try: - items = ctx.get_last_result_items() or [] - if 0 <= idx < len(items): - maybe = items[idx] - if isinstance(maybe, dict): - candidate = maybe.get("_selection_args") - if isinstance(candidate, (list, tuple)): - row_args = [str(x) for x in candidate if x is not None] - except Exception: - row_args = row_args or None - if row_args: - inserted = stages[0] - if inserted: - cmd = inserted[0] - tail = [str(x) for x in inserted[1:]] - stages[0] = [cmd] + [str(x) for x in row_args] + tail - except Exception: - pass - - # After inserting/appending an auto-stage, continue processing so later - # selection-expansion logic can still run (e.g., for example selectors). - return True, piped_result - else: - print("No previous results to select from\n") - return False, None - - return True, None - - @staticmethod - def _maybe_start_live_progress(config: Any, - stages: List[List[str]]) -> tuple[Any, - Dict[int, - int]]: - progress_ui = None - pipe_index_by_stage: Dict[int, - int] = {} - - try: - quiet_mode = ( - bool(config.get("_quiet_background_output")) - if isinstance(config, - dict) else False - ) - except Exception: - quiet_mode = False - - try: - import sys as _sys - - if (not quiet_mode) and bool(getattr(_sys.stderr, - "isatty", lambda: False)()): - from SYS.models import PipelineLiveProgress - - pipe_stage_indices: List[int] = [] - pipe_labels: List[str] = [] - for idx, stage_tokens in enumerate(stages): - if not stage_tokens: - continue - name = str(stage_tokens[0]).replace("_", "-").lower() - if name == "@" or name.startswith("@"): - continue - - # add-file directory selector stage: avoid Live progress so the - # selection table renders cleanly. - if name in {"add-file", - "add_file"}: - try: - from pathlib import Path as _Path - - toks = list(stage_tokens[1:]) - i = 0 - while i < len(toks): - t = str(toks[i]) - low = t.lower().strip() - if low in {"-path", - "--path", - "-p"} and i + 1 < len(toks): - nxt = str(toks[i + 1]) - if nxt and ("," not in nxt): - p = _Path(nxt) - if p.exists() and p.is_dir(): - name = "" # mark as skipped - break - i += 2 - continue - i += 1 - except Exception: - pass - if not name: - continue - # Display-only: avoid Live progress for relationship viewing. - # This keeps `@1 | get-relationship` clean and prevents progress UI - # from interfering with Rich tables/panels. - if name in {"get-relationship", - "get-rel"}: - continue - # `.pipe` (MPV) is an interactive launcher; disable pipeline Live progress - # for it because it doesn't meaningfully "complete" (mpv may keep running) - # and Live output interferes with MPV playlist UI. - if name in {".pipe", ".mpv"}: - continue - # `.matrix` uses a two-phase picker (@N then .matrix -send). Pipeline Live - # progress can linger across those phases and interfere with interactive output. - if name == ".matrix": - continue - # `delete-file` prints a Rich table directly; Live progress interferes and - # can truncate/overwrite the output. - if name in {"delete-file", - "del-file"}: - continue - pipe_stage_indices.append(idx) - pipe_labels.append(name) - - if pipe_labels: - progress_ui = PipelineLiveProgress(pipe_labels, enabled=True) - progress_ui.start() - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(progress_ui) - except Exception: - pass - pipe_index_by_stage = { - stage_idx: pipe_idx - for pipe_idx, stage_idx in enumerate(pipe_stage_indices) - } - except Exception: - progress_ui = None - pipe_index_by_stage = {} - - return progress_ui, pipe_index_by_stage - - def execute_tokens(self, tokens: List[str]) -> None: - from cmdlet import REGISTRY - from SYS import pipeline as ctx - - try: - self._try_clear_pipeline_stop(ctx) - - # REPL guard: stage-local tables should not persist across independent - # commands. Selection stages can always seed from last/display tables. - try: - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(None) - except Exception: - pass - - # Preflight (URL-duplicate prompts, etc.) should be cached within a single - # pipeline run, not across independent pipelines. - try: - ctx.store_value("preflight", - {}) - except Exception: - pass - - stages = self._split_stages(tokens) - if not stages: - print("Invalid pipeline syntax\n") - return - self._maybe_seed_current_stage_table(ctx) - stages = self._maybe_apply_pending_pipeline_tail(ctx, stages) - config = self._config_loader.load() - config = self._apply_quiet_background_flag(config) - - ( - stages, - first_stage_selection_indices, - first_stage_had_extra_args, - first_stage_select_all, - ) = self._extract_first_stage_selection_tokens(stages) - first_stage_selection_indices = self._apply_select_all_if_requested( - ctx, - first_stage_selection_indices, - first_stage_select_all - ) - - piped_result: Any = None - worker_manager = WorkerManagerRegistry.ensure(config) - pipeline_text = " | ".join(" ".join(stage) for stage in stages) - pipeline_session = WorkerStages.begin_pipeline( - worker_manager, - pipeline_text=pipeline_text, - config=config - ) - if pipeline_session and worker_manager: - self._log_pipeline_event( - worker_manager, - pipeline_session.worker_id, - f"Pipeline start: {pipeline_text or '(empty pipeline)'}", - ) - raw_stage_texts = self._get_raw_stage_texts(ctx) - self._maybe_enable_background_notifier( - worker_manager, - config, - pipeline_session - ) - - pipeline_status = "completed" - pipeline_error = "" - - progress_ui = None - pipe_index_by_stage: Dict[int, - int] = {} - - try: - ok, initial_piped = self._maybe_apply_initial_selection( - ctx, - config, - stages, - selection_indices=first_stage_selection_indices, - first_stage_had_extra_args=first_stage_had_extra_args, - worker_manager=worker_manager, - pipeline_session=pipeline_session, - ) - if not ok: - return - if initial_piped is not None: - piped_result = initial_piped - - # REPL guard: prevent add-relationship before add-file for download-file pipelines. - if not self._validate_download_file_relationship_order(stages): - pipeline_status = "failed" - pipeline_error = "Invalid pipeline order" - return - - # ------------------------------------------------------------------ - # Multi-level pipeline progress (pipes = stages, tasks = items) - # ------------------------------------------------------------------ - progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages) - - for stage_index, stage_tokens in enumerate(stages): - if not stage_tokens: - continue - - raw_stage_name = str(stage_tokens[0]) - cmd_name = raw_stage_name.replace("_", "-").lower() - stage_args = stage_tokens[1:] - - if cmd_name == "@": - # Prefer piping the last emitted/visible items (e.g. add-file results) - # over the result-table subject. The subject can refer to older context - # (e.g. a playlist row) and may not contain store+hash. - last_items = None - try: - last_items = ctx.get_last_result_items() - except Exception: - last_items = None - - if last_items: - from cmdlet._shared import coerce_to_pipe_object - - try: - pipe_items = [ - coerce_to_pipe_object(x) for x in list(last_items) - ] - except Exception: - pipe_items = list(last_items) - piped_result = pipe_items if len(pipe_items - ) > 1 else pipe_items[0] - try: - ctx.set_last_items(pipe_items) - except Exception: - pass - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used last result items" - ) - except Exception: - pass - continue - - subject = ctx.get_last_result_subject() - if subject is None: - print("No current result context available for '@'\n") - pipeline_status = "failed" - pipeline_error = "No result items/subject for @" - return - piped_result = subject - try: - subject_items = subject if isinstance(subject, - list) else [subject] - ctx.set_last_items(subject_items) - except Exception: - pass - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used current table subject" - ) - except Exception: - pass - continue - - if cmd_name.startswith("@"): # selection stage - selection_token = raw_stage_name - selection = SelectionSyntax.parse(selection_token) - filter_spec = SelectionFilterSyntax.parse(selection_token) - is_select_all = selection_token.strip() == "@*" - if selection is None and filter_spec is None and not is_select_all: - print(f"Invalid selection: {selection_token}\n") - pipeline_status = "failed" - pipeline_error = f"Invalid selection {selection_token}" - return - - selected_indices = [] - # Prefer selecting from the last selectable *table* (search/playlist) - # rather than from display-only emitted items, unless we're explicitly - # selecting from an overlay table. - display_table = None - try: - display_table = ( - ctx.get_display_table() - if hasattr(ctx, - "get_display_table") else None - ) - except Exception: - display_table = None - - stage_table = ctx.get_current_stage_table() - # Selection should operate on the table the user sees. - # If a display overlay table exists, force it as the current-stage table - # so provider selectors (e.g. tidal.album -> tracks) behave consistently. - try: - if display_table is not None and hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(display_table) - stage_table = display_table - except Exception: - pass - - if not stage_table and display_table is not None: - stage_table = display_table - if not stage_table: - stage_table = ctx.get_last_result_table() - - try: - if hasattr(ctx, "debug_table_state"): - ctx.debug_table_state(f"selection {selection_token}") - except Exception: - pass - - if display_table is not None and stage_table is display_table: - items_list = ctx.get_last_result_items() or [] - else: - if hasattr(ctx, "get_last_selectable_result_items"): - items_list = ctx.get_last_selectable_result_items( - ) or [] - else: - items_list = ctx.get_last_result_items() or [] - - if is_select_all: - selected_indices = list(range(len(items_list))) - elif filter_spec is not None: - selected_indices = [ - i for i, item in enumerate(items_list) - if SelectionFilterSyntax.matches(item, filter_spec) - ] - else: - selected_indices = sorted( - [i - 1 for i in selection] - ) # type: ignore[arg-type] - - resolved_items = items_list if items_list else [] - filtered = [ - resolved_items[i] for i in selected_indices - if 0 <= i < len(resolved_items) - ] - if not filtered: - print("No items matched selection\n") - pipeline_status = "failed" - pipeline_error = "Empty selection" - return - - # Filter UX: if the stage token is a filter and it's terminal, - # render a filtered table overlay rather than selecting/auto-downloading. - stage_is_last = (stage_index + 1 >= len(stages)) - if filter_spec is not None and stage_is_last: - try: - base_table = stage_table - if base_table is None: - base_table = ctx.get_last_result_table() - - if base_table is not None and hasattr(base_table, "copy_with_title"): - new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") - else: - new_table = ResultTable(getattr(base_table, "title", "") if base_table is not None else "Results") - - try: - if base_table is not None and getattr(base_table, "table", None): - new_table.set_table(str(getattr(base_table, "table"))) - except Exception: - pass - - try: - # Attach a one-line header so users see the active filter. - safe = str(selection_token)[1:].strip() - new_table.set_header_line(f'filter: "{safe}"') - except Exception: - pass - - for item in filtered: - new_table.add_result(item) - - try: - ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject()) - except Exception: - pass - - try: - stdout_console().print() - stdout_console().print(new_table) - except Exception: - pass - except Exception: - pass - continue - - # UX: selecting a single URL row from get-url tables should open it. - # Only do this when the selection stage is terminal to avoid surprising - # side-effects in pipelines like `@1 | download-file`. - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - if (not is_select_all) and (len(filtered) == 1): - try: - PipelineExecutor._maybe_open_url_selection( - current_table, - filtered, - stage_is_last=(stage_index + 1 >= len(stages)), - ) - except Exception: - pass - - if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(stage_index + 1 >= len(stages))): - return - - # Special case: selecting multiple tags from get-tag and piping into delete-tag - # should batch into a single operation (one backend call). - next_cmd = None - try: - if stage_index + 1 < len(stages) and stages[stage_index + - 1]: - next_cmd = str(stages[stage_index + 1][0] - ).replace("_", - "-").lower() - except Exception: - next_cmd = None - - def _is_tag_row(obj: Any) -> bool: - try: - if (hasattr(obj, - "__class__") - and obj.__class__.__name__ == "TagItem" - and hasattr(obj, - "tag_name")): - return True - except Exception: - pass - try: - if isinstance(obj, dict) and obj.get("tag_name"): - return True - except Exception: - pass - return False - - if (next_cmd in {"delete-tag", - "delete_tag"} and len(filtered) > 1 - and all(_is_tag_row(x) for x in filtered)): - from cmdlet._shared import get_field - - tags: List[str] = [] - first_hash = None - first_store = None - first_path = None - for item in filtered: - tag_name = get_field(item, "tag_name") - if tag_name: - tags.append(str(tag_name)) - if first_hash is None: - first_hash = get_field(item, "hash") - if first_store is None: - first_store = get_field(item, "store") - if first_path is None: - first_path = get_field(item, - "path") or get_field( - item, - "target" - ) - - if tags: - grouped = { - "table": "tag.selection", - "media_kind": "tag", - "hash": first_hash, - "store": first_store, - "path": first_path, - "tag": tags, - } - piped_result = grouped - continue - - from cmdlet._shared import coerce_to_pipe_object - - filtered_pipe_objs = [ - coerce_to_pipe_object(item) for item in filtered - ] - piped_result = ( - filtered_pipe_objs - if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] - ) - - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - table_type = ( - current_table.table - if current_table and hasattr(current_table, - "table") else None - ) - - def _norm_stage_cmd(name: Any) -> str: - return str(name or "").replace("_", "-").strip().lower() - - next_cmd = None - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = _norm_stage_cmd(stages[stage_index + 1][0]) - - auto_stage = None - if isinstance(table_type, str) and table_type: - try: - from ProviderCore.registry import selection_auto_stage_for_table - - # Preserve historical behavior: only forward selection-stage args - # to the auto stage when we are appending a new last stage. - at_end = bool(stage_index + 1 >= len(stages)) - auto_stage = selection_auto_stage_for_table( - table_type, - stage_args if at_end else None, - ) - except Exception: - auto_stage = None - - # Auto-insert downloader stages for provider tables. - # IMPORTANT: do not auto-download for filter selections; they may match many rows. - if filter_spec is None: - if stage_index + 1 >= len(stages): - if auto_stage: - try: - print(f"Auto-running selection via {auto_stage[0]}") - except Exception: - pass - stages.append(list(auto_stage)) - else: - if auto_stage: - auto_cmd = _norm_stage_cmd(auto_stage[0]) - if next_cmd not in (auto_cmd, ".pipe", ".mpv"): - debug(f"Auto-inserting {auto_cmd} after selection") - stages.insert(stage_index + 1, list(auto_stage)) - continue - - cmd_fn = REGISTRY.get(cmd_name) - if not cmd_fn: - try: - mod = import_cmd_module(cmd_name) - data = getattr(mod, "CMDLET", None) if mod else None - if data and hasattr(data, "exec") and callable(getattr(data, "exec")): - run_fn = getattr(data, "exec") - REGISTRY[cmd_name] = run_fn - cmd_fn = run_fn - except Exception: - cmd_fn = None - if not cmd_fn: - print(f"Unknown command: {cmd_name}\n") - pipeline_status = "failed" - pipeline_error = f"Unknown command: {cmd_name}" - return - - stage_session = WorkerStages.begin_stage( - worker_manager, - cmd_name=cmd_name, - stage_tokens=stage_tokens, - config=config, - command_text=" ".join(stage_tokens).strip(), - ) - - stage_worker_id = stage_session.worker_id if stage_session else None - stage_summary = self._summarize_stage_text(stage_tokens) - if pipeline_session and worker_manager: - summary_text = stage_summary or cmd_name - self._log_pipeline_event( - worker_manager, - pipeline_session.worker_id, - f"Stage {stage_index + 1}/{len(stages)} start: {summary_text}", - ) - - # Estimate how many per-item tasks this pipe will run. - pipe_idx = pipe_index_by_stage.get(stage_index) - if progress_ui is not None and pipe_idx is not None: - try: - # Prefer piped input for task counts. - if isinstance(piped_result, list): - total_items = len(piped_result) - preview_items: Optional[List[Any]] = list(piped_result) - elif piped_result is not None: - total_items = 1 - preview_items = [piped_result] - else: - # First stage without piped input: infer from URL-ish args. - preview: List[Any] = [] - - toks = list(stage_tokens[1:]) - i = 0 - while i < len(toks): - t = str(toks[i]) - low = t.lower().strip() - if (cmd_name == "add-file" and low in {"-path", - "--path", - "-p"} - and i + 1 < len(toks)): - nxt = str(toks[i + 1]) - if nxt: - if "," in nxt: - parts = [ - p.strip().strip("\"'") - for p in nxt.split(",") - ] - parts = [p for p in parts if p] - if parts: - preview.extend(parts) - i += 2 - continue - else: - preview.append(nxt) - i += 2 - continue - if low in {"-url", - "--url"} and i + 1 < len(toks): - nxt = str(toks[i + 1]) - if nxt and not nxt.startswith("-"): - preview.append(nxt) - i += 2 - continue - if (not t.startswith("-")) and ("://" in low - or low.startswith( - ("magnet:", - "torrent:"))): - preview.append(t) - i += 1 - - preview_items = preview if preview else None - total_items = len(preview) if preview else 1 - - progress_ui.begin_pipe( - pipe_idx, - total_items=int(total_items), - items_preview=preview_items - ) - except Exception: - pass - - on_emit = None - if progress_ui is not None and pipe_idx is not None: - _ui = cast(Any, progress_ui) - - def _on_emit( - obj: Any, - _idx: int = int(pipe_idx), - _progress=_ui - ) -> None: - try: - _progress.on_emit(_idx, obj) - except Exception: - pass - - on_emit = _on_emit - - pipeline_ctx = ctx.PipelineStageContext( - stage_index=stage_index, - total_stages=len(stages), - pipe_index=pipe_idx, - worker_id=stage_worker_id, - on_emit=on_emit, - ) - ctx.set_stage_context(pipeline_ctx) - stage_status = "completed" - stage_error = "" - - stage_label = f"stage {stage_index + 1}/{len(stages)} ({cmd_name})" - try: - # Avoid leaking interactive selection tables across stages. - # (Selection/expansion happens before this loop, so clearing here is safe.) - try: - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(None) - except Exception: - pass - - try: - if hasattr(ctx, "set_current_cmdlet_name"): - ctx.set_current_cmdlet_name(cmd_name) - except Exception: - pass - - try: - if hasattr(ctx, "set_current_stage_text"): - stage_text = "" - if raw_stage_texts and stage_index < len(raw_stage_texts - ): - candidate = str(raw_stage_texts[stage_index] - or "").strip() - if candidate: - try: - cand_tokens = shlex.split(candidate) - except Exception: - cand_tokens = candidate.split() - if cand_tokens: - first = str(cand_tokens[0] - ).replace("_", - "-").lower() - if first == cmd_name: - stage_text = candidate - if not stage_text: - stage_text = " ".join(stage_tokens).strip() - ctx.set_current_stage_text(stage_text) - except Exception: - pass - - # `.pipe`/`.mpv` is typically the terminal interactive stage (MPV UI). - # Stop Live progress before running it so output doesn't get stuck behind Live. - if (cmd_name in {".pipe", ".mpv"} and progress_ui is not None - and (stage_index + 1 >= len(stages))): - try: - progress_ui.stop() - except Exception: - pass - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(None) - except Exception: - pass - progress_ui = None - - ret_code = cmd_fn(piped_result, list(stage_args), config) - - stage_is_last = stage_index + 1 >= len(stages) - - # Graceful early-stop: preflight declined, etc. - try: - stop_req = ( - ctx.get_pipeline_stop() - if hasattr(ctx, - "get_pipeline_stop") else None - ) - except Exception: - stop_req = None - if stop_req is not None: - # Do not treat as an error; just end the pipeline quietly. - pipeline_status = "completed" - pipeline_error = "" - return - - emits: List[Any] = [] - if getattr(pipeline_ctx, "emits", None) is not None: - emits = list(pipeline_ctx.emits or []) - - # Shared `-path` behavior: persist temp/PATH artifacts to destination. - if emits: - try: - from cmdlet import _shared as sh - from SYS import models - - # 1. Apply -path persistence (moves temp files to final destination) - emits = sh.apply_output_path_from_pipeobjects( - cmd_name=cmd_name, - args=list(stage_args), - emits=emits, - ) - - # 2. METADATA STICKINESS / PROPAGATION - # We normalize all emitted items and merge metadata/tags from the previous stage. - # This ensures info like track titles/lyrics survive downloads/conversions. - # See cmdlet._shared.propagate_metadata for the merge logic. - prev_items = piped_result - if not isinstance(prev_items, (list, tuple)): - prev_items = [prev_items] if prev_items else [] - - emits = sh.propagate_metadata(prev_items, emits) - - try: - pipeline_ctx.emits = list(emits) - except Exception: - pass - except Exception: - pass - - if emits: - # If the cmdlet already installed an overlay table (e.g. get-tag), - # don't overwrite it: set_last_result_items_only() would clear the - # overlay table/subject and break '@' subject piping. - try: - has_overlay = ( - bool(ctx.get_display_table()) - if hasattr(ctx, - "get_display_table") else False - ) - except Exception: - has_overlay = False - if not has_overlay: - ctx.set_last_result_items_only(emits) - piped_result = emits - else: - piped_result = None - - # Some cmdlets (notably download-file format selection) populate a selectable - # current-stage table without emitting pipeline items. In these cases, render - # the table and pause the pipeline so the user can pick @N. - stage_table = ( - ctx.get_current_stage_table() - if hasattr(ctx, - "get_current_stage_table") else None - ) - stage_table_type = ( - str(getattr(stage_table, - "table", - "") or "").strip().lower() - if stage_table else "" - ) - try: - stage_table_source = ( - str(getattr(stage_table, - "source_command", - "") or "").strip().replace("_", - "-").lower() - if stage_table else "" - ) - except Exception: - stage_table_source = "" - if ((not stage_is_last) and (not emits) and cmd_name in { - "download-file", - "download-data", - "download_data", - } and stage_table is not None - and (stage_table_type in { - "ytdlp.formatlist", - "download-file", - "bandcamp", - "youtube", - } or stage_table_source in {"download-file"} - or stage_table_type in {"internetarchive.format", "internetarchive.formats"} - or stage_table_source in {"download-file"})): - try: - is_selectable = not bool( - getattr(stage_table, - "no_choice", - False) - ) - except Exception: - is_selectable = True - - if is_selectable: - try: - already_rendered = bool( - getattr( - stage_table, - "_rendered_by_cmdlet", - False - ) - ) - except Exception: - already_rendered = False - - if not already_rendered: - # Stop the Live progress display before printing a selectable table. - # Printing while Live is active can cause the table to be truncated/overwritten. - if progress_ui is not None: - try: - if pipe_idx is not None: - progress_ui.finish_pipe( - int(pipe_idx), - force_complete=True - ) - except Exception: - pass - try: - progress_ui.stop() - except Exception: - pass - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, - "set_live_progress"): - _pipeline_ctx.set_live_progress(None) - except Exception: - pass - progress_ui = None - stdout_console().print() - stdout_console().print(stage_table) - - # Always pause the pipeline when a selectable table was produced. - # The user will continue by running @N/@* which will re-attach the - # pending downstream stages. - - try: - remaining = stages[stage_index + 1:] - source_cmd = ( - ctx.get_current_stage_table_source_command() - if hasattr( - ctx, - "get_current_stage_table_source_command" - ) else None - ) - if remaining and hasattr( - ctx, - "set_pending_pipeline_tail"): - ctx.set_pending_pipeline_tail( - remaining, - source_command=source_cmd or cmd_name - ) - except Exception: - pass - return - - # For the final stage, many cmdlets rely on the runner to render the - # table they placed into pipeline context (e.g. get-tag). Prefer a - # display table if one exists, otherwise the current-stage table. - if stage_is_last: - # Stop the Live progress display before printing the final table. - # This avoids cursor-control interactions that can truncate output. - if progress_ui is not None: - try: - if pipe_idx is not None: - progress_ui.finish_pipe( - int(pipe_idx), - force_complete=( - stage_status == "completed" - ), - ) - except Exception: - pass - try: - progress_ui.stop() - except Exception: - pass - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(None) - except Exception: - pass - progress_ui = None - - final_table = None - try: - final_table = ( - ctx.get_display_table() - if hasattr(ctx, - "get_display_table") else None - ) - except Exception: - final_table = None - if final_table is None: - final_table = stage_table - - # If the cmdlet emitted results but didn't supply a fresh table, it's - # common for `stage_table` to still point at the previous stage's table - # (e.g. add-file's canonical store table). In that case, prefer rendering - # the emitted results so the user sees the actual output of this stage. - if (emits and (ctx.get_display_table() if hasattr( - ctx, - "get_display_table") else None) is None): - try: - src_cmd = ( - str( - getattr(final_table, - "source_command", - "") or "" - ).strip().lower() if final_table else "" - ) - except Exception: - src_cmd = "" - try: - cur_cmd = str(cmd_name - or "").strip().replace("_", - "-").lower() - except Exception: - cur_cmd = "" - if ((final_table is None) or (not src_cmd) - or (src_cmd.replace("_", - "-") != cur_cmd)): - try: - table_title = CmdletExecutor._get_table_title_for_command( - cmd_name, - emits, - list(stage_args) - ) - except Exception: - table_title = "Results" - table = ResultTable(table_title) - for item in emits: - table.add_result(item) - try: - if hasattr(ctx, - "set_last_result_table_overlay"): - ctx.set_last_result_table_overlay( - table, - emits - ) - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(table) - except Exception: - pass - final_table = table - - if final_table is not None: - try: - already_rendered = bool( - getattr( - final_table, - "_rendered_by_cmdlet", - False - ) - ) - except Exception: - already_rendered = False - - if not already_rendered: - stdout_console().print() - stdout_console().print(final_table) - - # (Fallback handled above by synthesizing an overlay ResultTable.) - - if isinstance(ret_code, int) and ret_code != 0: - stage_status = "failed" - stage_error = f"exit code {ret_code}" - print(f"[{stage_label} exit code: {ret_code}]\n") - pipeline_status = "failed" - pipeline_error = f"{stage_label} failed ({stage_error})" - return - except Exception as exc: - stage_status = "failed" - stage_error = f"{type(exc).__name__}: {exc}" - print(f"[error in {stage_label}]: {stage_error}\n") - pipeline_status = "failed" - pipeline_error = f"{stage_label} error: {exc}" - return - finally: - if pipeline_session and worker_manager: - status_label = ( - "completed" if stage_status == "completed" else "failed" - ) - msg = f"{stage_label} {status_label}" - if stage_error and stage_status != "completed": - msg += f": {stage_error}" - self._log_pipeline_event( - worker_manager, - pipeline_session.worker_id, - msg, - ) - if progress_ui is not None and pipe_idx is not None: - try: - progress_ui.finish_pipe( - int(pipe_idx), - force_complete=(stage_status == "completed") - ) - except Exception: - pass - try: - if hasattr(ctx, "clear_current_cmdlet_name"): - ctx.clear_current_cmdlet_name() - except Exception: - pass - try: - if hasattr(ctx, "clear_current_stage_text"): - ctx.clear_current_stage_text() - except Exception: - pass - if stage_session: - stage_session.close( - status=stage_status, - error_msg=stage_error - ) - elif pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - f"{stage_label} {'completed' if stage_status == 'completed' else 'failed'}", - ) - except Exception: - pass - - if not stages and piped_result is not None: - # Special-case: selecting metadata rows (e.g., get-tag -scrape) should - # immediately apply tags to the target item instead of just echoing a - # selection table. - try: - items = piped_result if isinstance(piped_result, list) else [piped_result] - applied_any = False - from cmdlet._shared import normalize_hash # type: ignore - from cmdlet.get_tag import _filter_scraped_tags, _emit_tags_as_table # type: ignore - from Store import Store # type: ignore - cfg_loader = ConfigLoader(root=Path.cwd()) - config = cfg_loader.load() - - for item in items: - if not isinstance(item, dict): - continue - provider = item.get("provider") - tags = item.get("tag") - if not provider or not isinstance(tags, list) or not tags: - continue - - file_hash = normalize_hash( - item.get("hash") - or item.get("hash_hex") - or item.get("file_hash") - or item.get("sha256") - ) - store_name = item.get("store") or item.get("storage") - subject_path = ( - item.get("path") - or item.get("target") - or item.get("filename") - ) - - if str(provider).strip().lower() == "ytdlp": - apply_tags = [str(t) for t in tags if t is not None] - else: - apply_tags = _filter_scraped_tags([str(t) for t in tags if t is not None]) - - if not apply_tags: - continue - - if store_name and file_hash: - try: - backend = Store(config)[str(store_name)] - backend.add_tag(file_hash, apply_tags, config=config) - try: - updated_tags, _src = backend.get_tag(file_hash, config=config) - except Exception: - updated_tags = apply_tags - _emit_tags_as_table( - tags_list=list(updated_tags or apply_tags), - file_hash=file_hash, - store=str(store_name), - service_name=None, - config=config, - item_title=str(item.get("title") or provider), - path=str(subject_path) if subject_path else None, - subject=item, - ) - applied_any = True - continue - except Exception: - pass - - # No store/hash: just emit the tags to the pipeline/view. - _emit_tags_as_table( - tags_list=list(apply_tags), - file_hash=file_hash, - store=str(store_name or "local"), - service_name=None, - config=config, - item_title=str(item.get("title") or provider), - path=str(subject_path) if subject_path else None, - subject=item, - ) - applied_any = True - - if applied_any: - # Selection handled; skip default selection echo. - return - except Exception: - # Fall back to default selection rendering on any failure. - pass - - items = piped_result if isinstance(piped_result, list) else [piped_result] - - # Special-case: selecting notes should show the text content directly. - note_like_items = [ - i for i in items - if isinstance(i, dict) and ("note_text" in i or "note" in i) - ] - if note_like_items: - for idx, item in enumerate(note_like_items, 1): - note_name = str( - item.get("note_name") - or item.get("name") - or f"note {idx}" - ).strip() - note_text = str(item.get("note_text") or item.get("note") or "") - note_text = note_text[:999] - stdout_console().print() - stdout_console().print(f"{note_name}:\n{note_text}") - ctx.set_last_result_items_only(items) - return - - # Special-case: selecting a single image should show it directly. - if len(items) == 1: - item = items[0] - - # Try to get hash and store to resolve through the backend - file_hash = None - store_name = None - if isinstance(item, dict): - file_hash = item.get("hash") - store_name = item.get("store") - else: - if hasattr(item, "hash"): - file_hash = getattr(item, "hash", None) - if hasattr(item, "store"): - store_name = getattr(item, "store", None) - - # Try to resolve the file through the Store backend if we have hash + store - resolved_file_path = None - if file_hash and store_name: - try: - from Store import Store - storage = Store(config=config or {}) - backend = storage[str(store_name)] - - # Call get_file to resolve the hash to an actual file path - maybe_path = backend.get_file(str(file_hash)) - - if isinstance(maybe_path, Path): - resolved_file_path = maybe_path - elif isinstance(maybe_path, str) and maybe_path: - # Only treat as a Path if it doesn't look like a URL - if not maybe_path.startswith(("http://", "https://")): - resolved_file_path = Path(maybe_path) - except Exception: - # Fallback: try using the path field from the item - pass - - # If backend resolution failed, try the path field - if not resolved_file_path: - path_str = None - if isinstance(item, dict): - path_str = ( - item.get("path") - or item.get("PATH") - or item.get("target") - or item.get("filename") - ) - else: - # Try attributes for PipeObject/SearchResult/etc. - for attr in ("path", "PATH", "target", "filename"): - if hasattr(item, attr): - val = getattr(item, attr) - if val and isinstance(val, (str, Path)): - path_str = val - break - - if path_str: - from SYS.utils import expand_path - resolved_file_path = expand_path(path_str).resolve() - - # Now check if it's an image and render it - is_image = False - if resolved_file_path: - try: - if resolved_file_path.suffix.lower() in IMAGE_EXTENSIONS and resolved_file_path.exists(): - # Use our image renderer - stdout_console().print() - render_image_to_console(resolved_file_path) - is_image = True - elif resolved_file_path.suffix.lower() in IMAGE_EXTENSIONS and not resolved_file_path.exists(): - stdout_console().print(f"[yellow]Warning: Image file not found at {resolved_file_path}[/yellow]") - except Exception: - pass - - # Render the comprehensive details panel for the item in either case - item_to_details = item if isinstance(item, dict) else ( - item.to_dict() if hasattr(item, "to_dict") else vars(item) - ) - # Ensure we include the resolved path if we found one - if resolved_file_path and "path" not in item_to_details: - item_to_details["path"] = str(resolved_file_path) - - render_item_details_panel(item_to_details) - - ctx.set_last_result_items_only(items) - return - - table = ResultTable("Selection Result") - for item in items: - table.add_result(item) - ctx.set_last_result_items_only(items) - stdout_console().print() - stdout_console().print(table) - except Exception as exc: - pipeline_status = "failed" - pipeline_error = str(exc) - print(f"[error] Failed to execute pipeline: {exc}\n") - finally: - if progress_ui is not None: - try: - progress_ui.stop() - except Exception: - pass - try: - from SYS import pipeline as _pipeline_ctx - - if hasattr(_pipeline_ctx, "set_live_progress"): - _pipeline_ctx.set_live_progress(None) - except Exception: - pass - # End-of-command cleanup: avoid leaking current stage tables into - # the next REPL command (causes stale @ selection sources). - try: - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(None) - except Exception: - pass - if pipeline_session and worker_manager: - final_msg = f"Pipeline {pipeline_status}" - if pipeline_error: - final_msg += f": {pipeline_error}" - else: - final_msg += " (ok)" - self._log_pipeline_event( - worker_manager, - pipeline_session.worker_id, - final_msg, - ) - if pipeline_session: - pipeline_session.close( - status=pipeline_status, - error_msg=pipeline_error - ) - except Exception as exc: - print(f"[error] Failed to execute pipeline: {exc}\n") - - -from rich.markdown import Markdown -from rich.console import Console console = Console() diff --git a/MPV/mpv_ipc.py b/MPV/mpv_ipc.py index db012a0..a90782c 100644 --- a/MPV/mpv_ipc.py +++ b/MPV/mpv_ipc.py @@ -351,10 +351,10 @@ class MPV: pipeline += f" | add-file -path {_q(path or '')}" try: - from TUI.pipeline_runner import PipelineExecutor # noqa: WPS433 + from TUI.pipeline_runner import PipelineRunner # noqa: WPS433 - executor = PipelineExecutor() - result = executor.run_pipeline(pipeline) + runner = PipelineRunner() + result = runner.run_pipeline(pipeline) return { "success": bool(getattr(result, "success", diff --git a/MPV/pipeline_helper.py b/MPV/pipeline_helper.py index 6e6cd8a..0d686ad 100644 --- a/MPV/pipeline_helper.py +++ b/MPV/pipeline_helper.py @@ -74,7 +74,7 @@ OBS_ID_REQUEST = 1001 def _run_pipeline(pipeline_text: str, *, seeds: Any = None) -> Dict[str, Any]: # Import after sys.path fix. - from TUI.pipeline_runner import PipelineExecutor # noqa: WPS433 + from TUI.pipeline_runner import PipelineRunner # noqa: WPS433 def _table_to_payload(table: Any) -> Optional[Dict[str, Any]]: if table is None: @@ -133,8 +133,8 @@ def _run_pipeline(pipeline_text: str, *, seeds: Any = None) -> Dict[str, Any]: "rows": rows_payload } - executor = PipelineExecutor() - result = executor.run_pipeline(pipeline_text, seeds=seeds) + runner = PipelineRunner() + result = runner.run_pipeline(pipeline_text, seeds=seeds) table_payload = None try: @@ -905,7 +905,7 @@ def main(argv: Optional[list[str]] = None) -> int: ] ) _append_helper_log( - f"[helper] published store-choices to user-data/medeia-store-choices-cached" + "[helper] published store-choices to user-data/medeia-store-choices-cached" ) except Exception as exc: _append_helper_log( diff --git a/Provider/HIFI.py b/Provider/HIFI.py index 1c0dc08..597af42 100644 --- a/Provider/HIFI.py +++ b/Provider/HIFI.py @@ -1,15 +1,12 @@ from __future__ import annotations -import os -import random import re import shutil -import string import subprocess import time import sys from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse from API.Tidal import ( @@ -20,7 +17,6 @@ from API.Tidal import ( stringify, ) from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments -from ProviderCore.inline_utils import collect_choice from cmdlet._shared import get_field from SYS import pipeline as pipeline_context from SYS.logger import debug, log diff --git a/Provider/Tidal.py b/Provider/Tidal.py index 60f0181..afd2cf1 100644 --- a/Provider/Tidal.py +++ b/Provider/Tidal.py @@ -1,15 +1,12 @@ from __future__ import annotations -import os -import random import re import shutil -import string import subprocess import time import sys from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse from API.Tidal import ( diff --git a/Provider/libgen.py b/Provider/libgen.py index 0e2e046..f93dc0d 100644 --- a/Provider/libgen.py +++ b/Provider/libgen.py @@ -1265,7 +1265,7 @@ class LibgenSearch: _call(log_info, f"[libgen] Using mirror: {mirror}") return results else: - _call(log_info, f"[libgen] Mirror returned 0 results; stopping mirror fallback") + _call(log_info, "[libgen] Mirror returned 0 results; stopping mirror fallback") break except requests.exceptions.Timeout: _call(log_info, f"[libgen] Mirror timed out: {mirror}") diff --git a/Provider/openlibrary.py b/Provider/openlibrary.py index 5cd5c1c..3e9a6cb 100644 --- a/Provider/openlibrary.py +++ b/Provider/openlibrary.py @@ -11,7 +11,7 @@ import sys import tempfile import time from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple from urllib.parse import urlparse import requests @@ -20,7 +20,7 @@ from API.HTTP import HTTPClient, get_requests_verify_value from ProviderCore.base import Provider, SearchResult from SYS.utils import sanitize_filename from SYS.cli_syntax import get_field, get_free_text, parse_query -from SYS.logger import debug, log +from SYS.logger import log from Provider.metadata_provider import ( archive_item_metadata_to_tags, fetch_archive_item_metadata, diff --git a/Provider/youtube.py b/Provider/youtube.py index 68ee1a1..e946346 100644 --- a/Provider/youtube.py +++ b/Provider/youtube.py @@ -109,7 +109,6 @@ class YouTube(TableProviderMixin, Provider): def validate(self) -> bool: try: - import yt_dlp # type: ignore return True except Exception: diff --git a/Provider/ytdlp.py b/Provider/ytdlp.py index b0ea6c9..ff61ad8 100644 --- a/Provider/ytdlp.py +++ b/Provider/ytdlp.py @@ -9,13 +9,11 @@ This keeps format selection logic in ytdlp and leaves add-file plug-and-play. from __future__ import annotations -import sys from typing import Any, Dict, Iterable, List, Optional, Tuple from ProviderCore.base import Provider, SearchResult from SYS.provider_helpers import TableProviderMixin -from SYS.logger import log, debug -from tool.ytdlp import list_formats, is_url_supported_by_ytdlp +from SYS.logger import debug class ytdlp(TableProviderMixin, Provider): @@ -196,7 +194,6 @@ class ytdlp(TableProviderMixin, Provider): def validate(self) -> bool: """Validate yt-dlp availability.""" try: - import yt_dlp # type: ignore return True except Exception: return False @@ -295,7 +292,7 @@ try: debug(f"[ytdlp] Selection routed with format_id: {format_id}") return result_args - debug(f"[ytdlp] Warning: No selection args or format_id found in row") + debug("[ytdlp] Warning: No selection args or format_id found in row") return [] register_provider( diff --git a/ProviderCore/base.py b/ProviderCore/base.py index 80fe301..79b0720 100644 --- a/ProviderCore/base.py +++ b/ProviderCore/base.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from abc import ABC, abstractmethod +from abc import ABC from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple, Callable diff --git a/SYS/background_services.py b/SYS/background_services.py index b07923d..04bdc41 100644 --- a/SYS/background_services.py +++ b/SYS/background_services.py @@ -1,9 +1,7 @@ from __future__ import annotations -import os import sys import subprocess -import atexit from pathlib import Path from typing import Optional diff --git a/SYS/metadata.py b/SYS/metadata.py index a676aeb..3aeed40 100644 --- a/SYS/metadata.py +++ b/SYS/metadata.py @@ -4,13 +4,10 @@ import subprocess import sys import shutil from SYS.logger import log, debug -from urllib.parse import urlsplit, urlunsplit, unquote -from collections import deque from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple from API.HydrusNetwork import apply_hydrus_tag_mutation, fetch_hydrus_metadata, fetch_hydrus_metadata_by_url -from SYS.models import FileRelationshipTracker try: # Optional; used when available for richer metadata fetches import yt_dlp @@ -2585,7 +2582,7 @@ def scrape_url_metadata( ) except json_module.JSONDecodeError: pass - except Exception as e: + except Exception: pass # Silently ignore if we can't get playlist entries # Fallback: if still no tags detected, get from first item diff --git a/SYS/optional_deps.py b/SYS/optional_deps.py index e34142e..cef870c 100644 --- a/SYS/optional_deps.py +++ b/SYS/optional_deps.py @@ -4,7 +4,7 @@ import importlib import os import subprocess import sys -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, List, Tuple from SYS.logger import log from SYS.rich_display import stdout_console diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 5f0c1e5..be01148 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -4,13 +4,22 @@ Pipeline execution context and state management for cmdlet. from __future__ import annotations import sys -import shlex from contextlib import contextmanager from dataclasses import dataclass, field from contextvars import ContextVar from typing import Any, Dict, List, Optional, Sequence from SYS.models import PipelineStageContext from SYS.logger import log, debug, is_debug_enabled +from SYS.worker import WorkerManagerRegistry, WorkerStages +from SYS.cli_parsing import SelectionSyntax, SelectionFilterSyntax +from SYS.rich_display import stdout_console +from SYS.background_notifier import ensure_background_notifier +from SYS.result_table import Table + +HELP_EXAMPLE_SOURCE_COMMANDS = { + ".help-example", + "help-example", +} def set_live_progress(progress_ui: Any) -> None: @@ -1076,3 +1085,1683 @@ def clear_last_result() -> None: state.last_result_table = None state.last_result_items = [] state.last_result_subject = None + + +def _split_pipeline_tokens(tokens: Sequence[str]) -> List[List[str]]: + """Split example tokens into per-stage command sequences using pipe separators.""" + stages: List[List[str]] = [] + current: List[str] = [] + for token in tokens: + if token == "|": + if current: + stages.append(current) + current = [] + continue + current.append(str(token)) + if current: + stages.append(current) + return [stage for stage in stages if stage] + + +class PipelineExecutor: + + def __init__(self, *, config_loader: Optional[Any] = None) -> None: + self._config_loader = config_loader + self._toolbar_output: Optional[Callable[[str], None]] = None + + def _load_config(self) -> Dict[str, Any]: + try: + if self._config_loader is not None: + return self._config_loader.load() + except Exception: + pass + try: + from SYS.config import load_config + + return load_config() + except Exception: + return {} + + def set_toolbar_output(self, output: Optional[Callable[[str], None]]) -> None: + self._toolbar_output = output + + @staticmethod + def _split_stages(tokens: Sequence[str]) -> List[List[str]]: + stages: List[List[str]] = [] + current: List[str] = [] + for token in tokens: + if token == "|": + if current: + stages.append(current) + current = [] + else: + current.append(token) + if current: + stages.append(current) + return stages + + @staticmethod + def _validate_download_file_relationship_order(stages: List[List[str]]) -> bool: + """Guard against running add-relationship on unstored download-file results. + + Intended UX: + download-file ... | add-file -store | add-relationship + + Rationale: + download-file outputs items that may not yet have a stable store+hash. + add-relationship is designed to operate in store/hash mode. + """ + + def _norm(name: str) -> str: + return str(name or "").replace("_", "-").strip().lower() + + names: List[str] = [] + for stage in stages or []: + if not stage: + continue + names.append(_norm(stage[0])) + + dl_idxs = [i for i, n in enumerate(names) if n == "download-file"] + rel_idxs = [i for i, n in enumerate(names) if n == "add-relationship"] + add_file_idxs = [i for i, n in enumerate(names) if n == "add-file"] + + if not dl_idxs or not rel_idxs: + return True + + # If download-file is upstream of add-relationship, require an add-file in between. + for rel_i in rel_idxs: + dl_before = [d for d in dl_idxs if d < rel_i] + if not dl_before: + continue + dl_i = max(dl_before) + if not any(dl_i < a < rel_i for a in add_file_idxs): + print( + "Pipeline order error: when using download-file with add-relationship, " + "add-relationship must come after add-file (so items are stored and have store+hash).\n" + "Example: download-file <...> | add-file -store | add-relationship\n" + ) + return False + + return True + + @staticmethod + def _try_clear_pipeline_stop(ctx: Any) -> None: + try: + if hasattr(ctx, "clear_pipeline_stop"): + ctx.clear_pipeline_stop() + except Exception: + pass + + @staticmethod + def _maybe_seed_current_stage_table(ctx: Any) -> None: + try: + if hasattr(ctx, + "get_current_stage_table") and not ctx.get_current_stage_table(): + display_table = ( + ctx.get_display_table() if hasattr(ctx, + "get_display_table") else None + ) + if display_table: + ctx.set_current_stage_table(display_table) + else: + last_table = ( + ctx.get_last_result_table() + if hasattr(ctx, + "get_last_result_table") else None + ) + if last_table: + ctx.set_current_stage_table(last_table) + except Exception: + pass + + @staticmethod + def _maybe_apply_pending_pipeline_tail(ctx: Any, + stages: List[List[str]]) -> List[List[str]]: + try: + pending_tail = ( + ctx.get_pending_pipeline_tail() + if hasattr(ctx, + "get_pending_pipeline_tail") else [] + ) + pending_source = ( + ctx.get_pending_pipeline_source() + if hasattr(ctx, + "get_pending_pipeline_source") else None + ) + except Exception: + pending_tail = [] + pending_source = None + + try: + current_source = ( + ctx.get_current_stage_table_source_command() + if hasattr(ctx, + "get_current_stage_table_source_command") else None + ) + except Exception: + current_source = None + + try: + effective_source = current_source or ( + ctx.get_last_result_table_source_command() + if hasattr(ctx, + "get_last_result_table_source_command") else None + ) + except Exception: + effective_source = current_source + + selection_start = bool( + stages and stages[0] and stages[0][0].startswith("@") + ) + + def _tail_is_suffix(existing: List[List[str]], tail: List[List[str]]) -> bool: + if not tail or not existing: + return False + if len(tail) > len(existing): + return False + return existing[-len(tail):] == tail + + if pending_tail and selection_start: + if (pending_source is None) or (effective_source + and pending_source == effective_source): + # Only append the pending tail if the user hasn't already provided it. + if not _tail_is_suffix(stages, pending_tail): + stages = list(stages) + list(pending_tail) + try: + if hasattr(ctx, "clear_pending_pipeline_tail"): + ctx.clear_pending_pipeline_tail() + except Exception: + pass + else: + try: + if hasattr(ctx, "clear_pending_pipeline_tail"): + ctx.clear_pending_pipeline_tail() + except Exception: + pass + return stages + + def _apply_quiet_background_flag(self, config: Any) -> Any: + if isinstance(config, dict): + # This executor is used by both the REPL and the `pipeline` subcommand. + # Quiet/background mode is helpful for detached/background runners, but + # it suppresses interactive UX (like the pipeline Live progress UI). + config["_quiet_background_output"] = bool(self._toolbar_output is None) + return config + + @staticmethod + def _extract_first_stage_selection_tokens( + stages: List[List[str]], + ) -> tuple[List[List[str]], + List[int], + bool, + bool]: + first_stage_tokens = stages[0] if stages else [] + first_stage_selection_indices: List[int] = [] + first_stage_had_extra_args = False + first_stage_select_all = False + + if first_stage_tokens: + new_first_stage: List[str] = [] + for token in first_stage_tokens: + if token.startswith("@"): # selection + selection = SelectionSyntax.parse(token) + if selection is not None: + first_stage_selection_indices = sorted( + [i - 1 for i in selection] + ) + continue + if token == "@*": + first_stage_select_all = True + continue + new_first_stage.append(token) + + if new_first_stage: + stages = list(stages) + stages[0] = new_first_stage + if first_stage_selection_indices or first_stage_select_all: + first_stage_had_extra_args = True + elif first_stage_selection_indices or first_stage_select_all: + stages = list(stages) + stages.pop(0) + + return ( + stages, + first_stage_selection_indices, + first_stage_had_extra_args, + first_stage_select_all, + ) + + @staticmethod + def _apply_select_all_if_requested(ctx: Any, + indices: List[int], + select_all: bool) -> List[int]: + if not select_all: + return indices + try: + last_items = ctx.get_last_result_items() + except Exception: + last_items = None + if last_items: + return list(range(len(last_items))) + return indices + + @staticmethod + def _maybe_run_class_selector( + ctx: Any, + config: Any, + selected_items: list, + *, + stage_is_last: bool + ) -> bool: + if not stage_is_last: + return False + + candidates: list[str] = [] + seen: set[str] = set() + + def _add(value) -> None: + try: + text = str(value or "").strip().lower() + except Exception: + return + if not text or text in seen: + return + seen.add(text) + candidates.append(text) + + try: + current_table = ctx.get_current_stage_table() or ctx.get_last_result_table() + _add( + current_table. + table if current_table and hasattr(current_table, + "table") else None + ) + + # Prefer an explicit provider hint from table metadata when available. + # This keeps @N selectors working even when row payloads don't carry a + # provider key (or when they carry a table-type like tidal.album). + try: + meta = ( + current_table.get_table_metadata() + if current_table is not None and hasattr(current_table, "get_table_metadata") + else getattr(current_table, "table_metadata", None) + ) + except Exception: + meta = None + if isinstance(meta, dict): + _add(meta.get("provider")) + except Exception: + pass + + for item in selected_items or []: + if isinstance(item, dict): + _add(item.get("provider")) + _add(item.get("store")) + _add(item.get("table")) + else: + _add(getattr(item, "provider", None)) + _add(getattr(item, "store", None)) + _add(getattr(item, "table", None)) + + try: + from ProviderCore.registry import get_provider, is_known_provider_name + except Exception: + get_provider = None # type: ignore + is_known_provider_name = None # type: ignore + + # If we have a table-type like "tidal.album", also try its provider prefix ("tidal") + # when that prefix is a registered provider name. + if is_known_provider_name is not None: + try: + for key in list(candidates): + if not isinstance(key, str): + continue + if "." not in key: + continue + if is_known_provider_name(key): + continue + prefix = str(key).split(".", 1)[0].strip().lower() + if prefix and is_known_provider_name(prefix): + _add(prefix) + except Exception: + pass + + if get_provider is not None: + for key in candidates: + try: + if is_known_provider_name is not None and ( + not is_known_provider_name(key)): + continue + except Exception: + # If the predicate fails for any reason, fall back to legacy behavior. + pass + try: + provider = get_provider(key, config) + except Exception: + continue + selector = getattr(provider, "selector", None) + if selector is None: + continue + try: + handled = bool( + selector(selected_items, + ctx=ctx, + stage_is_last=True) + ) + except Exception as exc: + print(f"{key} selector failed: {exc}\n") + return True + if handled: + return True + + store_keys: list[str] = [] + for item in selected_items or []: + if isinstance(item, dict): + v = item.get("store") + else: + v = getattr(item, "store", None) + name = str(v or "").strip() + if name: + store_keys.append(name) + + if store_keys: + try: + from Store.registry import Store as StoreRegistry + + store_registry = StoreRegistry(config, suppress_debug=True) + _backend_names = list(store_registry.list_backends() or []) + _backend_by_lower = { + str(n).lower(): str(n) + for n in _backend_names if str(n).strip() + } + for name in store_keys: + resolved_name = name + if not store_registry.is_available(resolved_name): + resolved_name = _backend_by_lower.get(str(name).lower(), name) + if not store_registry.is_available(resolved_name): + continue + backend = store_registry[resolved_name] + selector = getattr(backend, "selector", None) + if selector is None: + continue + handled = bool( + selector(selected_items, + ctx=ctx, + stage_is_last=True) + ) + if handled: + return True + except Exception: + pass + + return False + + @staticmethod + def _summarize_stage_text(stage_tokens: Sequence[str], limit: int = 140) -> str: + combined = " ".join(str(tok) for tok in stage_tokens if tok is not None).strip() + if not combined: + return "" + normalized = re.sub(r"\s+", " ", combined) + if len(normalized) <= limit: + return normalized + return normalized[:limit - 3].rstrip() + "..." + + @staticmethod + def _log_pipeline_event( + worker_manager: Any, + worker_id: Optional[str], + message: str, + ) -> None: + if not worker_manager or not worker_id or not message: + return + try: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + except Exception: + timestamp = "" + if timestamp: + text = f"{timestamp} - PIPELINE - {message}" + else: + text = f"PIPELINE - {message}" + try: + worker_manager.append_stdout(worker_id, text + "\n", channel="log") + except Exception: + pass + + @staticmethod + def _maybe_open_url_selection( + current_table: Any, + selected_items: list, + *, + stage_is_last: bool + ) -> bool: + if not stage_is_last: + return False + if not selected_items or len(selected_items) != 1: + return False + + table_type = "" + source_cmd = "" + try: + table_type = str(getattr(current_table, "table", "") or "").strip().lower() + except Exception: + table_type = "" + try: + source_cmd = ( + str(getattr(current_table, + "source_command", + "") or "").strip().replace("_", + "-").lower() + ) + except Exception: + source_cmd = "" + + if table_type != "url" and source_cmd != "get-url": + return False + + item = selected_items[0] + url = None + try: + from cmdlet._shared import get_field + + url = get_field(item, "url") + except Exception: + try: + url = item.get("url") if isinstance(item, + dict + ) else getattr(item, + "url", + None) + except Exception: + url = None + + url_text = str(url or "").strip() + if not url_text: + return False + + try: + import webbrowser + + webbrowser.open(url_text, new=2) + return True + except Exception: + return False + + def _maybe_enable_background_notifier( + self, + worker_manager: Any, + config: Any, + pipeline_session: Any + ) -> None: + if not (pipeline_session and worker_manager and isinstance(config, dict)): + return + + session_worker_ids = config.get("_session_worker_ids") + if not session_worker_ids: + return + + try: + output_fn = self._toolbar_output + quiet_mode = bool(config.get("_quiet_background_output")) + terminal_only = quiet_mode and not output_fn + kwargs: Dict[str, + Any] = { + "session_worker_ids": session_worker_ids, + "only_terminal_updates": terminal_only, + "overlay_mode": bool(output_fn), + } + if output_fn: + kwargs["output"] = output_fn + ensure_background_notifier(worker_manager, **kwargs) + except Exception: + pass + + @staticmethod + def _get_raw_stage_texts(ctx: Any) -> List[str]: + raw_stage_texts: List[str] = [] + try: + if hasattr(ctx, "get_current_command_stages"): + raw_stage_texts = ctx.get_current_command_stages() or [] + except Exception: + raw_stage_texts = [] + return raw_stage_texts + + def _maybe_apply_initial_selection( + self, + ctx: Any, + config: Any, + stages: List[List[str]], + *, + selection_indices: List[int], + first_stage_had_extra_args: bool, + worker_manager: Any, + pipeline_session: Any, + ) -> tuple[bool, + Any]: + if not selection_indices: + return True, None + + # Selection should operate on the *currently displayed* selectable table. + # Some navigation flows (e.g. @.. back) can show a display table without + # updating current_stage_table. Provider selectors rely on current_stage_table + # to detect table type (e.g. tidal.album -> tracks), so sync it here. + display_table = None + try: + display_table = ( + ctx.get_display_table() if hasattr(ctx, "get_display_table") else None + ) + except Exception: + display_table = None + + current_stage_table = None + try: + current_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") else None + ) + except Exception: + current_stage_table = None + + try: + if display_table is not None and hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(display_table) + elif current_stage_table is None and hasattr(ctx, "set_current_stage_table"): + last_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") else None + ) + if last_table is not None: + ctx.set_current_stage_table(last_table) + except Exception: + pass + + source_cmd = None + source_args_raw = None + try: + source_cmd = ctx.get_current_stage_table_source_command() + source_args_raw = ctx.get_current_stage_table_source_args() + except Exception: + source_cmd = None + source_args_raw = None + + if isinstance(source_args_raw, str): + source_args: List[str] = [source_args_raw] + elif isinstance(source_args_raw, list): + source_args = [str(x) for x in source_args_raw if x is not None] + else: + source_args = [] + + current_table = None + try: + current_table = ctx.get_current_stage_table() + except Exception: + current_table = None + table_type = ( + current_table.table if current_table and hasattr(current_table, + "table") else None + ) + + command_expanded = False + example_selector_triggered = False + normalized_source_cmd = str(source_cmd or "").replace("_", "-").strip().lower() + + if normalized_source_cmd in HELP_EXAMPLE_SOURCE_COMMANDS and selection_indices: + try: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + except Exception: + row_args = None + tokens: List[str] = [] + if isinstance(row_args, list) and row_args: + tokens = [str(x) for x in row_args if x is not None] + if tokens: + stage_groups = _split_pipeline_tokens(tokens) + if stage_groups: + for stage in reversed(stage_groups): + stages.insert(0, stage) + selection_indices = [] + command_expanded = True + example_selector_triggered = True + + if not example_selector_triggered: + if table_type in {"youtube", + "soulseek"}: + command_expanded = False + elif source_cmd == "search-file" and source_args and "youtube" in source_args: + command_expanded = False + else: + selected_row_args: List[str] = [] + skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0 + # Command expansion via @N: + # - Default behavior: expand ONLY for single-row selections. + # - Special case: allow multi-row expansion for add-file directory tables by + # combining selected rows into a single `-path file1,file2,...` argument. + if source_cmd and not skip_pipe_expansion: + src = str(source_cmd).replace("_", "-").strip().lower() + + if src == "add-file" and selection_indices: + row_args_list: List[List[str]] = [] + for idx in selection_indices: + try: + row_args = ctx.get_current_stage_table_row_selection_args( + idx + ) + except Exception: + row_args = None + if isinstance(row_args, list) and row_args: + row_args_list.append( + [str(x) for x in row_args if x is not None] + ) + + # Combine `['-path', ]` from each row into one `-path` arg. + paths: List[str] = [] + can_merge = bool(row_args_list) and ( + len(row_args_list) == len(selection_indices) + ) + if can_merge: + for ra in row_args_list: + if len(ra) == 2 and str(ra[0]).strip().lower() in { + "-path", + "--path", + "-p", + }: + p = str(ra[1]).strip() + if p: + paths.append(p) + else: + can_merge = False + break + + if can_merge and paths: + selected_row_args.extend(["-path", ",".join(paths)]) + elif len(selection_indices) == 1 and row_args_list: + selected_row_args.extend(row_args_list[0]) + else: + # Only perform @N command expansion for *single-item* selections. + # For multi-item selections (e.g. @*, @1-5), expanding to one row + # would silently drop items. In those cases we pipe items downstream. + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + if row_args: + selected_row_args.extend(row_args) + + if selected_row_args: + if isinstance(source_cmd, list): + cmd_list: List[str] = [str(x) for x in source_cmd if x is not None] + elif isinstance(source_cmd, str): + cmd_list = [source_cmd] + else: + cmd_list = [] + + # IMPORTANT: Put selected row args *before* source_args. + # Rationale: The cmdlet argument parser treats the *first* unknown + # token as a positional value (e.g., URL). If `source_args` + # contain unknown flags (like -provider which download-file does + # not declare), they could be misinterpreted as the positional + # URL argument and cause attempts to download strings like + # "-provider" (which is invalid). By placing selection args + # first we ensure the intended URL/selection token is parsed + # as the positional URL and avoid this class of parsing errors. + expanded_stage: List[str] = cmd_list + selected_row_args + source_args + + if first_stage_had_extra_args and stages: + expanded_stage += stages[0] + stages[0] = expanded_stage + else: + stages.insert(0, expanded_stage) + + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", + ) + except Exception: + pass + + stage_table = None + try: + stage_table = ctx.get_current_stage_table() + except Exception: + stage_table = None + + display_table = None + try: + display_table = ( + ctx.get_display_table() if hasattr(ctx, + "get_display_table") else None + ) + except Exception: + display_table = None + + if not stage_table and display_table is not None: + stage_table = display_table + if not stage_table: + try: + stage_table = ctx.get_last_result_table() + except Exception: + stage_table = None + + # Prefer selecting from the last selectable *table* (search/playlist) + # rather than from display-only emitted items, unless we're explicitly + # selecting from an overlay table. + try: + if display_table is not None and stage_table is display_table: + items_list = ctx.get_last_result_items() or [] + else: + if hasattr(ctx, "get_last_selectable_result_items"): + items_list = ctx.get_last_selectable_result_items() or [] + else: + items_list = ctx.get_last_result_items() or [] + except Exception: + items_list = [] + + resolved_items = items_list if items_list else [] + if items_list: + filtered = [ + resolved_items[i] for i in selection_indices + if 0 <= i < len(resolved_items) + ] + if not filtered: + print("No items matched selection in pipeline\n") + return False, None + + # Provider selection expansion (non-terminal): allow certain provider tables + # (e.g. tidal.album) to expand to multiple downstream items when the user + # pipes into another stage (e.g. @N | .mpv or @N | add-file). + table_type_hint = None + try: + table_type_hint = ( + stage_table.table + if stage_table is not None and hasattr(stage_table, "table") + else None + ) + except Exception: + table_type_hint = None + + if stages and isinstance(table_type_hint, str) and table_type_hint.strip().lower() == "tidal.album": + try: + from ProviderCore.registry import get_provider + + prov = get_provider("tidal", config) + except Exception: + prov = None + + if prov is not None and hasattr(prov, "_extract_album_selection_context") and hasattr(prov, "_tracks_for_album"): + try: + album_contexts = prov._extract_album_selection_context(filtered) # type: ignore[attr-defined] + except Exception: + album_contexts = [] + + track_items: List[Any] = [] + seen_track_ids: set[int] = set() + for album_id, album_title, artist_name in album_contexts or []: + try: + track_results = prov._tracks_for_album( # type: ignore[attr-defined] + album_id=album_id, + album_title=album_title, + artist_name=artist_name, + limit=500, + ) + except Exception: + track_results = [] + for tr in track_results or []: + try: + md = getattr(tr, "full_metadata", None) + tid = None + if isinstance(md, dict): + raw_id = md.get("trackId") or md.get("id") + try: + tid = int(raw_id) if raw_id is not None else None + except Exception: + tid = None + if tid is not None: + if tid in seen_track_ids: + continue + seen_track_ids.add(tid) + except Exception: + pass + track_items.append(tr) + + if track_items: + filtered = track_items + table_type_hint = "tidal.track" + + if PipelineExecutor._maybe_run_class_selector( + ctx, + config, + filtered, + stage_is_last=(not stages)): + return False, None + + from cmdlet._shared import coerce_to_pipe_object + + filtered_pipe_objs = [coerce_to_pipe_object(item) for item in filtered] + piped_result = ( + filtered_pipe_objs + if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] + ) + + if pipeline_session and worker_manager: + try: + selection_parts = [f"@{i+1}" for i in selection_indices] + worker_manager.log_step( + pipeline_session.worker_id, + f"Applied @N selection {' | '.join(selection_parts)}", + ) + except Exception: + pass + + # Auto-insert downloader stages for provider tables. + try: + current_table = ctx.get_current_stage_table() + if current_table is None and hasattr(ctx, "get_display_table"): + current_table = ctx.get_display_table() + if current_table is None: + current_table = ctx.get_last_result_table() + except Exception: + current_table = None + table_type = None + try: + if isinstance(table_type_hint, str) and table_type_hint.strip(): + table_type = table_type_hint + else: + table_type = ( + current_table.table + if current_table and hasattr(current_table, "table") else None + ) + except Exception: + table_type = ( + current_table.table + if current_table and hasattr(current_table, "table") else None + ) + + def _norm_cmd(name: Any) -> str: + return str(name or "").replace("_", "-").strip().lower() + + auto_stage = None + if isinstance(table_type, str) and table_type: + try: + from ProviderCore.registry import selection_auto_stage_for_table + + auto_stage = selection_auto_stage_for_table(table_type) + except Exception: + auto_stage = None + + source_cmd_for_selection = None + source_args_for_selection: List[str] = [] + try: + source_cmd_for_selection = ( + ctx.get_current_stage_table_source_command() + or ctx.get_last_result_table_source_command() + ) + source_args_for_selection = ( + ctx.get_current_stage_table_source_args() + or ctx.get_last_result_table_source_args() + or [] + ) + except Exception: + source_cmd_for_selection = None + source_args_for_selection = [] + + if not stages and selection_indices and source_cmd_for_selection: + src_norm = _norm_cmd(source_cmd_for_selection) + if src_norm in {".worker", "worker", "workers"}: + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = None + try: + row_args = ctx.get_current_stage_table_row_selection_args(idx) + except Exception: + row_args = None + if not row_args: + try: + row_args = ctx.get_last_result_table_row_selection_args(idx) + except Exception: + row_args = None + if not row_args: + try: + items = ctx.get_last_result_items() or [] + if 0 <= idx < len(items): + maybe = items[idx] + if isinstance(maybe, dict): + candidate = maybe.get("_selection_args") + if isinstance(candidate, (list, tuple)): + row_args = [str(x) for x in candidate if x is not None] + except Exception: + row_args = row_args or None + + if row_args: + stages.append( + [str(source_cmd_for_selection)] + + [str(x) for x in row_args if x is not None] + + [str(x) for x in source_args_for_selection if x is not None] + ) + + def _apply_row_action_to_stage(stage_idx: int) -> bool: + if not selection_indices or len(selection_indices) != 1: + return False + try: + row_action = ctx.get_current_stage_table_row_selection_action( + selection_indices[0] + ) + except Exception: + row_action = None + if not row_action: + # Fallback to serialized payload when the table row is unavailable + try: + items = ctx.get_last_result_items() or [] + if 0 <= selection_indices[0] < len(items): + maybe = items[selection_indices[0]] + if isinstance(maybe, dict): + candidate = maybe.get("_selection_action") + if isinstance(candidate, (list, tuple)): + row_action = [str(x) for x in candidate if x is not None] + debug(f"@N row {selection_indices[0]} restored action from payload: {row_action}") + except Exception: + row_action = row_action or None + if not row_action: + debug(f"@N row {selection_indices[0]} has no selection_action") + return False + normalized = [str(x) for x in row_action if x is not None] + if not normalized: + return False + debug(f"Applying row action for row {selection_indices[0]} -> {normalized}") + if 0 <= stage_idx < len(stages): + debug(f"Replacing stage {stage_idx} {stages[stage_idx]} with row action {normalized}") + stages[stage_idx] = normalized + return True + return False + + if not stages: + if isinstance(table_type, str) and table_type.startswith("metadata."): + print("Auto-applying metadata selection via get-tag") + stages.append(["get-tag"]) + elif auto_stage: + try: + print(f"Auto-running selection via {auto_stage[0]}") + except Exception: + pass + # Append the auto stage now. If the user also provided a selection + # (e.g., @1 | add-file ...), we want to attach the row selection + # args *to the auto-inserted stage* so the download command receives + # the selected row information immediately. + stages.append(list(auto_stage)) + debug(f"Inserted auto stage before row action: {stages[-1]}") + + # If the caller included a selection (e.g., @1) try to attach + # the selection args immediately to the inserted auto stage so + # the expansion is effective in a single pass. + if selection_indices: + try: + if not _apply_row_action_to_stage(len(stages) - 1): + # Only support single-row selection for auto-attach here + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + if not row_args: + try: + items = ctx.get_last_result_items() or [] + if 0 <= idx < len(items): + maybe = items[idx] + if isinstance(maybe, dict): + candidate = maybe.get("_selection_args") + if isinstance(candidate, (list, tuple)): + row_args = [str(x) for x in candidate if x is not None] + except Exception: + row_args = row_args or None + if row_args: + # Place selection args before any existing source args + inserted = stages[-1] + if inserted: + cmd = inserted[0] + tail = [str(x) for x in inserted[1:]] + stages[-1] = [cmd] + [str(x) for x in row_args] + tail + except Exception: + pass + else: + first_cmd = stages[0][0] if stages and stages[0] else None + if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in ( + "get-tag", + "get_tag", + ".pipe", + ".mpv", + ): + print("Auto-inserting get-tag after metadata selection") + stages.insert(0, ["get-tag"]) + elif auto_stage: + first_cmd_norm = _norm_cmd(first_cmd) + auto_cmd_norm = _norm_cmd(auto_stage[0]) + if first_cmd_norm not in (auto_cmd_norm, ".pipe", ".mpv"): + debug(f"Auto-inserting {auto_cmd_norm} after selection") + # Insert the auto stage before the user-specified stage + # Note: Do NOT append source_args here - they are search tokens from + # the previous stage and should not be passed to the downloader. + stages.insert(0, list(auto_stage)) + debug(f"Inserted auto stage before existing pipeline: {stages[0]}") + + # If a selection is present, attach the row selection args to the + # newly-inserted stage so the download stage runs with the + # selected row information. + if selection_indices: + try: + if not _apply_row_action_to_stage(0): + if len(selection_indices) == 1: + idx = selection_indices[0] + row_args = ctx.get_current_stage_table_row_selection_args(idx) + if not row_args: + try: + items = ctx.get_last_result_items() or [] + if 0 <= idx < len(items): + maybe = items[idx] + if isinstance(maybe, dict): + candidate = maybe.get("_selection_args") + if isinstance(candidate, (list, tuple)): + row_args = [str(x) for x in candidate if x is not None] + except Exception: + row_args = row_args or None + if row_args: + inserted = stages[0] + if inserted: + cmd = inserted[0] + tail = [str(x) for x in inserted[1:]] + stages[0] = [cmd] + [str(x) for x in row_args] + tail + except Exception: + pass + + # After inserting/appending an auto-stage, continue processing so later + # selection-expansion logic can still run (e.g., for example selectors). + return True, piped_result + else: + print("No previous results to select from\n") + return False, None + + return True, None + + @staticmethod + def _maybe_start_live_progress(config: Any, + stages: List[List[str]]) -> tuple[Any, + Dict[int, + int]]: + progress_ui = None + pipe_index_by_stage: Dict[int, + int] = {} + + try: + quiet_mode = ( + bool(config.get("_quiet_background_output")) + if isinstance(config, + dict) else False + ) + except Exception: + quiet_mode = False + + try: + import sys as _sys + + if (not quiet_mode) and bool(getattr(_sys.stderr, + "isatty", lambda: False)()): + from SYS.models import PipelineLiveProgress + + pipe_stage_indices: List[int] = [] + pipe_labels: List[str] = [] + for idx, stage_tokens in enumerate(stages): + if not stage_tokens: + continue + name = str(stage_tokens[0]).replace("_", "-").lower() + if name == "@" or name.startswith("@"): + continue + + # add-file directory selector stage: avoid Live progress so the + # selection table renders cleanly. + if name in {"add-file", + "add_file"}: + try: + from pathlib import Path as _Path + + toks = list(stage_tokens[1:]) + i = 0 + while i < len(toks): + t = str(toks[i]) + low = t.lower().strip() + if low in {"-path", + "--path", + "-p"} and i + 1 < len(toks): + nxt = str(toks[i + 1]) + if nxt and ("," not in nxt): + p = _Path(nxt) + if p.exists() and p.is_dir(): + name = "" # mark as skipped + break + i += 2 + continue + i += 1 + except Exception: + pass + if not name: + continue + # Display-only: avoid Live progress for relationship viewing. + # This keeps `@1 | get-relationship` clean and prevents progress UI + # from interfering with Rich tables/panels. + if name in {"get-relationship", + "get-rel"}: + continue + # `.pipe` (MPV) is an interactive launcher; disable pipeline Live progress + # for it because it doesn't meaningfully "complete" (mpv may keep running) + # and Live output interferes with MPV playlist UI. + if name in {".pipe", ".mpv"}: + continue + # `.matrix` uses a two-phase picker (@N then .matrix -send). Pipeline Live + # progress can linger across those phases and interfere with interactive output. + if name == ".matrix": + continue + # `delete-file` prints a Rich table directly; Live progress interferes and + # can truncate/overwrite the output. + if name in {"delete-file", + "del-file"}: + continue + pipe_stage_indices.append(idx) + pipe_labels.append(name) + + if pipe_labels: + progress_ui = PipelineLiveProgress(pipe_labels, enabled=True) + progress_ui.start() + try: + from SYS import pipeline as _pipeline_ctx + + if hasattr(_pipeline_ctx, "set_live_progress"): + _pipeline_ctx.set_live_progress(progress_ui) + except Exception: + pass + pipe_index_by_stage = { + stage_idx: pipe_idx + for pipe_idx, stage_idx in enumerate(pipe_stage_indices) + } + except Exception: + progress_ui = None + pipe_index_by_stage = {} + + return progress_ui, pipe_index_by_stage + + def execute_tokens(self, tokens: List[str]) -> None: + from cmdlet import REGISTRY + ctx = sys.modules[__name__] + + try: + self._try_clear_pipeline_stop(ctx) + + # REPL guard: stage-local tables should not persist across independent + # commands. Selection stages can always seed from last/display tables. + try: + if hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(None) + except Exception: + pass + + # Preflight (URL-duplicate prompts, etc.) should be cached within a single + # pipeline run, not across independent pipelines. + try: + ctx.store_value("preflight", + {}) + except Exception: + pass + + stages = self._split_stages(tokens) + if not stages: + print("Invalid pipeline syntax\n") + return + self._maybe_seed_current_stage_table(ctx) + stages = self._maybe_apply_pending_pipeline_tail(ctx, stages) + config = self._load_config() + config = self._apply_quiet_background_flag(config) + + ( + stages, + first_stage_selection_indices, + first_stage_had_extra_args, + first_stage_select_all, + ) = self._extract_first_stage_selection_tokens(stages) + first_stage_selection_indices = self._apply_select_all_if_requested( + ctx, + first_stage_selection_indices, + first_stage_select_all + ) + + piped_result: Any = None + worker_manager = WorkerManagerRegistry.ensure(config) + pipeline_text = " | ".join(" ".join(stage) for stage in stages) + pipeline_session = WorkerStages.begin_pipeline( + worker_manager, + pipeline_text=pipeline_text, + config=config + ) + if pipeline_session and worker_manager: + self._log_pipeline_event( + worker_manager, + pipeline_session.worker_id, + f"Pipeline start: {pipeline_text or '(empty pipeline)'}", + ) + raw_stage_texts = self._get_raw_stage_texts(ctx) + self._maybe_enable_background_notifier( + worker_manager, + config, + pipeline_session + ) + + pipeline_status = "completed" + pipeline_error = "" + + progress_ui = None + pipe_index_by_stage: Dict[int, + int] = {} + + try: + ok, initial_piped = self._maybe_apply_initial_selection( + ctx, + config, + stages, + selection_indices=first_stage_selection_indices, + first_stage_had_extra_args=first_stage_had_extra_args, + worker_manager=worker_manager, + pipeline_session=pipeline_session, + ) + if not ok: + return + if initial_piped is not None: + piped_result = initial_piped + except Exception as exc: + pipeline_status = "failed" + pipeline_error = f"{type(exc).__name__}: {exc}" + print(f"[error] {type(exc).__name__}: {exc}\n") + return + + # REPL guard: prevent add-relationship before add-file for download-file pipelines. + if not self._validate_download_file_relationship_order(stages): + pipeline_status = "failed" + pipeline_error = "Invalid pipeline order" + return + + # ------------------------------------------------------------------ + # Multi-level pipeline progress (pipes = stages, tasks = items) + # ------------------------------------------------------------------ + progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages) + + for stage_index, stage_tokens in enumerate(stages): + if not stage_tokens: + continue + + raw_stage_name = str(stage_tokens[0]) + cmd_name = raw_stage_name.replace("_", "-").lower() + stage_args = stage_tokens[1:] + + if cmd_name == "@": + # Prefer piping the last emitted/visible items (e.g. add-file results) + # over the result-table subject. The subject can refer to older context + # (e.g. a playlist row) and may not contain store+hash. + last_items = None + try: + last_items = ctx.get_last_result_items() + except Exception: + last_items = None + + if last_items: + from cmdlet._shared import coerce_to_pipe_object + + try: + pipe_items = [ + coerce_to_pipe_object(x) for x in list(last_items) + ] + except Exception: + pipe_items = list(last_items) + piped_result = pipe_items if len(pipe_items + ) > 1 else pipe_items[0] + try: + ctx.set_last_items(pipe_items) + except Exception: + pass + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + "@ used last result items" + ) + except Exception: + pass + continue + + subject = ctx.get_last_result_subject() + if subject is None: + print("No current result context available for '@'\n") + pipeline_status = "failed" + pipeline_error = "No result items/subject for @" + return + piped_result = subject + try: + subject_items = subject if isinstance(subject, + list) else [subject] + ctx.set_last_items(subject_items) + except Exception: + pass + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + "@ used current table subject" + ) + except Exception: + pass + continue + + if cmd_name.startswith("@"): # selection stage + selection_token = raw_stage_name + selection = SelectionSyntax.parse(selection_token) + filter_spec = SelectionFilterSyntax.parse(selection_token) + is_select_all = selection_token.strip() == "@*" + if selection is None and filter_spec is None and not is_select_all: + print(f"Invalid selection: {selection_token}\n") + pipeline_status = "failed" + pipeline_error = f"Invalid selection {selection_token}" + return + + selected_indices = [] + # Prefer selecting from the last selectable *table* (search/playlist) + # rather than from display-only emitted items, unless we're explicitly + # selecting from an overlay table. + display_table = None + try: + display_table = ( + ctx.get_display_table() + if hasattr(ctx, + "get_display_table") else None + ) + except Exception: + display_table = None + + stage_table = ctx.get_current_stage_table() + # Selection should operate on the table the user sees. + # If a display overlay table exists, force it as the current-stage table + # so provider selectors (e.g. tidal.album -> tracks) behave consistently. + try: + if display_table is not None and hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(display_table) + stage_table = display_table + except Exception: + pass + + if not stage_table and display_table is not None: + stage_table = display_table + if not stage_table: + stage_table = ctx.get_last_result_table() + + try: + if hasattr(ctx, "debug_table_state"): + ctx.debug_table_state(f"selection {selection_token}") + except Exception: + pass + + if display_table is not None and stage_table is display_table: + items_list = ctx.get_last_result_items() or [] + else: + if hasattr(ctx, "get_last_selectable_result_items"): + items_list = ctx.get_last_selectable_result_items( + ) or [] + else: + items_list = ctx.get_last_result_items() or [] + + if is_select_all: + selected_indices = list(range(len(items_list))) + elif filter_spec is not None: + selected_indices = [ + i for i, item in enumerate(items_list) + if SelectionFilterSyntax.matches(item, filter_spec) + ] + else: + selected_indices = sorted( + [i - 1 for i in selection] + ) # type: ignore[arg-type] + + resolved_items = items_list if items_list else [] + filtered = [ + resolved_items[i] for i in selected_indices + if 0 <= i < len(resolved_items) + ] + if not filtered: + print("No items matched selection\n") + pipeline_status = "failed" + pipeline_error = "Empty selection" + return + + # Filter UX: if the stage token is a filter and it's terminal, + # render a filtered table overlay rather than selecting/auto-downloading. + stage_is_last = (stage_index + 1 >= len(stages)) + if filter_spec is not None and stage_is_last: + try: + base_table = stage_table + if base_table is None: + base_table = ctx.get_last_result_table() + + if base_table is not None and hasattr(base_table, "copy_with_title"): + new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") + else: + new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results") + + try: + if base_table is not None and getattr(base_table, "table", None): + new_table.set_table(str(getattr(base_table, "table"))) + except Exception: + pass + + try: + # Attach a one-line header so users see the active filter. + safe = str(selection_token)[1:].strip() + new_table.set_header_line(f'filter: "{safe}"') + except Exception: + pass + + for item in filtered: + new_table.add_result(item) + + try: + ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject()) + except Exception: + pass + + try: + stdout_console().print() + stdout_console().print(new_table) + except Exception: + pass + except Exception: + pass + continue + + # UX: selecting a single URL row from get-url tables should open it. + # Only do this when the selection stage is terminal to avoid surprising + # side-effects in pipelines like `@1 | download-file`. + current_table = ctx.get_current_stage_table( + ) or ctx.get_last_result_table() + if (not is_select_all) and (len(filtered) == 1): + try: + PipelineExecutor._maybe_open_url_selection( + current_table, + filtered, + stage_is_last=(stage_index + 1 >= len(stages)), + ) + except Exception: + pass + + if PipelineExecutor._maybe_run_class_selector( + ctx, + config, + filtered, + stage_is_last=(stage_index + 1 >= len(stages))): + return + + # Special case: selecting multiple tags from get-tag and piping into delete-tag + # should batch into a single operation (one backend call). + next_cmd = None + try: + if stage_index + 1 < len(stages) and stages[stage_index + 1]: + next_cmd = str(stages[stage_index + 1][0] + ).replace("_", + "-").lower() + except Exception: + next_cmd = None + + def _is_tag_row(obj: Any) -> bool: + try: + if (hasattr(obj, + "__class__") + and obj.__class__.__name__ == "TagItem" + and hasattr(obj, + "tag_name")): + return True + except Exception: + pass + try: + if isinstance(obj, dict) and obj.get("tag_name"): + return True + except Exception: + pass + return False + + if (next_cmd in {"delete-tag", + "delete_tag"} and len(filtered) > 1 + and all(_is_tag_row(x) for x in filtered)): + from cmdlet._shared import get_field + + tags: List[str] = [] + first_hash = None + first_store = None + first_path = None + for item in filtered: + tag_name = get_field(item, "tag_name") + if tag_name: + tags.append(str(tag_name)) + if first_hash is None: + first_hash = get_field(item, "hash") + if first_store is None: + first_store = get_field(item, "store") + if first_path is None: + first_path = get_field(item, + "path") or get_field( + item, + "target" + ) + + if tags: + grouped = { + "table": "tag.selection", + "media_kind": "tag", + "hash": first_hash, + "store": first_store, + "path": first_path, + "tag": tags, + } + piped_result = grouped + continue + + from cmdlet._shared import coerce_to_pipe_object + + filtered_pipe_objs = [ + coerce_to_pipe_object(item) for item in filtered + ] + piped_result = ( + filtered_pipe_objs + if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] + ) + + current_table = ctx.get_current_stage_table( + ) or ctx.get_last_result_table() + table_type = ( + current_table.table + if current_table and hasattr(current_table, + "table") else None + ) + + def _norm_stage_cmd(name: Any) -> str: + return str(name or "").replace("_", "-").strip().lower() + + next_cmd = None + if stage_index + 1 < len(stages) and stages[stage_index + 1]: + next_cmd = _norm_stage_cmd(stages[stage_index + 1][0]) + + auto_stage = None + if isinstance(table_type, str) and table_type: + try: + from ProviderCore.registry import selection_auto_stage_for_table + + # Preserve historical behavior: only forward selection-stage args + # to the auto stage when we are appending a new last stage. + at_end = bool(stage_index + 1 >= len(stages)) + auto_stage = selection_auto_stage_for_table( + table_type, + stage_args if at_end else None, + ) + except Exception: + auto_stage = None + + # Auto-insert downloader stages for provider tables. + # IMPORTANT: do not auto-download for filter selections; they may match many rows. + if filter_spec is None: + if stage_index + 1 >= len(stages): + if auto_stage: + try: + print(f"Auto-running selection via {auto_stage[0]}") + except Exception: + pass + stages.append(list(auto_stage)) + else: + if auto_stage: + auto_cmd = _norm_stage_cmd(auto_stage[0]) + if next_cmd not in (auto_cmd, ".pipe", ".mpv"): + debug(f"Auto-inserting {auto_cmd} after selection") + stages.insert(stage_index + 1, list(auto_stage)) + continue + + cmd_fn = REGISTRY.get(cmd_name) + if not cmd_fn: + try: + mod = import_cmd_module(cmd_name) + data = getattr(mod, "CMDLET", None) if mod else None + if data and hasattr(data, "exec") and callable(getattr(data, "exec")): + run_fn = getattr(data, "exec") + REGISTRY[cmd_name] = run_fn + cmd_fn = run_fn + except Exception: + cmd_fn = None + + if not cmd_fn: + print(f"Unknown command: {cmd_name}\n") + pipeline_status = "failed" + pipeline_error = f"Unknown command: {cmd_name}" + return + except Exception as exc: + pipeline_status = "failed" + pipeline_error = f"{type(exc).__name__}: {exc}" + print(f"[error] {type(exc).__name__}: {exc}\n") + finally: + # Stop Live progress and clear pipeline-level live progress + if progress_ui is not None: + try: + progress_ui.stop() + except Exception: + pass + try: + from SYS import pipeline as _pipeline_ctx + if hasattr(_pipeline_ctx, "set_live_progress"): + _pipeline_ctx.set_live_progress(None) + except Exception: + pass + # Close pipeline session and log final status + try: + if pipeline_session and worker_manager: + pipeline_session.close(status=pipeline_status, error_msg=pipeline_error) + except Exception: + pass + try: + if pipeline_session and worker_manager: + self._log_pipeline_event(worker_manager, pipeline_session.worker_id, + f"Pipeline {pipeline_status}: {pipeline_error or ''}") + except Exception: + pass diff --git a/SYS/provider_helpers.py b/SYS/provider_helpers.py index c40dca5..0eea87c 100644 --- a/SYS/provider_helpers.py +++ b/SYS/provider_helpers.py @@ -18,8 +18,7 @@ so authors don't have to install pandas/bs4 unless they want to. """ from __future__ import annotations -from typing import Any, Dict, List, Optional -from urllib.parse import quote_plus +from typing import List, Optional from API.HTTP import HTTPClient from ProviderCore.base import SearchResult diff --git a/SYS/result_table.py b/SYS/result_table.py index 01788fe..916bfd5 100644 --- a/SYS/result_table.py +++ b/SYS/result_table.py @@ -16,7 +16,6 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Callable, Set from pathlib import Path import json -import shutil from rich.box import SIMPLE from rich.console import Group @@ -1678,7 +1677,7 @@ class Table: try: int(value) except ValueError: - print(f"Must be an integer") + print("Must be an integer") continue return value diff --git a/SYS/rich_display.py b/SYS/rich_display.py index 7e2148e..6af8d33 100644 --- a/SYS/rich_display.py +++ b/SYS/rich_display.py @@ -11,7 +11,7 @@ from __future__ import annotations import contextlib import sys -from typing import Any, Iterator, Sequence, TextIO +from typing import Any, Iterator, TextIO from rich.console import Console from rich.panel import Panel @@ -81,7 +81,6 @@ def show_provider_config_panel( ) -> None: """Show a Rich panel explaining how to configure providers.""" from rich.table import Table as RichTable - from rich.text import Text from rich.console import Group if isinstance(provider_names, str): @@ -117,7 +116,6 @@ def show_store_config_panel( ) -> None: """Show a Rich panel explaining how to configure storage backends.""" from rich.table import Table as RichTable - from rich.text import Text from rich.console import Group if isinstance(store_names, str): @@ -152,7 +150,6 @@ def show_available_providers_panel(provider_names: List[str]) -> None: """Show a Rich panel listing available/configured providers.""" from rich.columns import Columns from rich.console import Group - from rich.text import Text if not provider_names: return diff --git a/SYS/utils.py b/SYS/utils.py index fd86557..f8d1461 100644 --- a/SYS/utils.py +++ b/SYS/utils.py @@ -14,9 +14,8 @@ except Exception: import os import base64 import logging -import time from pathlib import Path -from typing import Any, Iterable, Optional +from typing import Any, Iterable from datetime import datetime from dataclasses import dataclass, field from fnmatch import fnmatch diff --git a/SYS/worker.py b/SYS/worker.py new file mode 100644 index 0000000..5accfbc --- /dev/null +++ b/SYS/worker.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import atexit +import io +import sys +import uuid +from pathlib import Path +from typing import Any, Dict, Optional, Set, TextIO + +from SYS.config import get_local_storage_path +from SYS.worker_manager import WorkerManager + + +class WorkerOutputMirror(io.TextIOBase): + """Mirror stdout/stderr to worker manager while preserving console output.""" + + def __init__( + self, + original: TextIO, + manager: WorkerManager, + worker_id: str, + channel: str, + ): + self._original = original + self._manager = manager + self._worker_id = worker_id + self._channel = channel + self._pending: str = "" + + def write(self, data: str) -> int: # type: ignore[override] + if not data: + return 0 + self._original.write(data) + self._buffer_text(data) + return len(data) + + def flush(self) -> None: # type: ignore[override] + self._original.flush() + self._flush_pending(force=True) + + def isatty(self) -> bool: # pragma: no cover + return bool(getattr(self._original, "isatty", lambda: False)()) + + def _buffer_text(self, data: str) -> None: + combined = self._pending + data + lines = combined.splitlines(keepends=True) + if not lines: + self._pending = combined + return + + if lines[-1].endswith(("\n", "\r")): + complete = lines + self._pending = "" + else: + complete = lines[:-1] + self._pending = lines[-1] + + for chunk in complete: + self._emit(chunk) + + def _flush_pending(self, *, force: bool = False) -> None: + if self._pending and force: + self._emit(self._pending) + self._pending = "" + + def _emit(self, text: str) -> None: + if not text: + return + try: + self._manager.append_stdout(self._worker_id, text, channel=self._channel) + except Exception: + pass + + @property + def encoding(self) -> str: # type: ignore[override] + return getattr(self._original, "encoding", "utf-8") + + +class WorkerStageSession: + """Lifecycle helper for wrapping a CLI cmdlet execution in a worker record.""" + + def __init__( + self, + *, + manager: WorkerManager, + worker_id: str, + orig_stdout: TextIO, + orig_stderr: TextIO, + stdout_proxy: WorkerOutputMirror, + stderr_proxy: WorkerOutputMirror, + config: Optional[Dict[str, Any]], + logging_enabled: bool, + completion_label: str, + error_label: str, + ) -> None: + self.manager = manager + self.worker_id = worker_id + self.orig_stdout = orig_stdout + self.orig_stderr = orig_stderr + self.stdout_proxy = stdout_proxy + self.stderr_proxy = stderr_proxy + self.config = config + self.logging_enabled = logging_enabled + self.closed = False + self._completion_label = completion_label + self._error_label = error_label + + def close(self, *, status: str = "completed", error_msg: str = "") -> None: + if self.closed: + return + try: + self.stdout_proxy.flush() + self.stderr_proxy.flush() + except Exception: + pass + + sys.stdout = self.orig_stdout + sys.stderr = self.orig_stderr + + if self.logging_enabled: + try: + self.manager.disable_logging_for_worker(self.worker_id) + except Exception: + pass + + try: + if status == "completed": + self.manager.log_step(self.worker_id, self._completion_label) + else: + self.manager.log_step( + self.worker_id, f"{self._error_label}: {error_msg or status}" + ) + except Exception: + pass + + try: + self.manager.finish_worker( + self.worker_id, result=status or "completed", error_msg=error_msg or "" + ) + except Exception: + pass + + if self.config and self.config.get("_current_worker_id") == self.worker_id: + self.config.pop("_current_worker_id", None) + self.closed = True + + +class WorkerManagerRegistry: + """Process-wide WorkerManager cache keyed by library_root.""" + + _manager: Optional[WorkerManager] = None + _manager_root: Optional[Path] = None + _orphan_cleanup_done: bool = False + _registered: bool = False + + @classmethod + def ensure(cls, config: Dict[str, Any]) -> Optional[WorkerManager]: + if not isinstance(config, dict): + return None + + existing = config.get("_worker_manager") + if isinstance(existing, WorkerManager): + return existing + + library_root = get_local_storage_path(config) + if not library_root: + return None + + try: + resolved_root = Path(library_root).resolve() + except Exception: + resolved_root = Path(library_root) + + try: + if cls._manager is None or cls._manager_root != resolved_root: + if cls._manager is not None: + try: + cls._manager.close() + except Exception: + pass + cls._manager = WorkerManager(resolved_root, auto_refresh_interval=0.5) + cls._manager_root = resolved_root + + manager = cls._manager + config["_worker_manager"] = manager + + if manager is not None and not cls._orphan_cleanup_done: + try: + manager.expire_running_workers( + older_than_seconds=120, + worker_id_prefix="cli_%", + reason=( + "CLI session ended unexpectedly; marking worker as failed", + ), + ) + except Exception: + pass + else: + cls._orphan_cleanup_done = True + + if not cls._registered: + atexit.register(cls.close) + cls._registered = True + + return manager + except Exception as exc: + print(f"[worker] Could not initialize worker manager: {exc}", file=sys.stderr) + return None + + @classmethod + def close(cls) -> None: + if cls._manager is None: + return + try: + cls._manager.close() + except Exception: + pass + cls._manager = None + cls._manager_root = None + cls._orphan_cleanup_done = False + + +class WorkerStages: + """Factory methods for stage/pipeline worker sessions.""" + + @staticmethod + def _start_worker_session( + worker_manager: Optional[WorkerManager], + *, + worker_type: str, + title: str, + description: str, + pipe_text: str, + config: Optional[Dict[str, Any]], + completion_label: str, + error_label: str, + skip_logging_for: Optional[Set[str]] = None, + session_worker_ids: Optional[Set[str]] = None, + ) -> Optional[WorkerStageSession]: + if worker_manager is None: + return None + if skip_logging_for and worker_type in skip_logging_for: + return None + + safe_type = worker_type or "cmd" + worker_id = f"cli_{safe_type[:8]}_{uuid.uuid4().hex[:6]}" + + try: + tracked = worker_manager.track_worker( + worker_id, + worker_type=worker_type, + title=title, + description=description or "(no args)", + pipe=pipe_text, + ) + if not tracked: + return None + except Exception as exc: + print(f"[worker] Failed to track {worker_type}: {exc}", file=sys.stderr) + return None + + if session_worker_ids is not None: + session_worker_ids.add(worker_id) + + logging_enabled = False + try: + handler = worker_manager.enable_logging_for_worker(worker_id) + logging_enabled = handler is not None + except Exception: + logging_enabled = False + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + stdout_proxy = WorkerOutputMirror(orig_stdout, worker_manager, worker_id, "stdout") + stderr_proxy = WorkerOutputMirror(orig_stderr, worker_manager, worker_id, "stderr") + sys.stdout = stdout_proxy + sys.stderr = stderr_proxy + if isinstance(config, dict): + config["_current_worker_id"] = worker_id + + try: + worker_manager.log_step(worker_id, f"Started {worker_type}") + except Exception: + pass + + return WorkerStageSession( + manager=worker_manager, + worker_id=worker_id, + orig_stdout=orig_stdout, + orig_stderr=orig_stderr, + stdout_proxy=stdout_proxy, + stderr_proxy=stderr_proxy, + config=config, + logging_enabled=logging_enabled, + completion_label=completion_label, + error_label=error_label, + ) + + @classmethod + def begin_stage( + cls, + worker_manager: Optional[WorkerManager], + *, + cmd_name: str, + stage_tokens: Sequence[str], + config: Optional[Dict[str, Any]], + command_text: str, + ) -> Optional[WorkerStageSession]: + description = " ".join(stage_tokens[1:]) if len(stage_tokens) > 1 else "(no args)" + session_worker_ids = None + if isinstance(config, dict): + session_worker_ids = config.get("_session_worker_ids") + + return cls._start_worker_session( + worker_manager, + worker_type=cmd_name, + title=f"{cmd_name} stage", + description=description, + pipe_text=command_text, + config=config, + completion_label="Stage completed", + error_label="Stage error", + skip_logging_for={".worker", "worker", "workers"}, + session_worker_ids=session_worker_ids, + ) + + @classmethod + def begin_pipeline( + cls, + worker_manager: Optional[WorkerManager], + *, + pipeline_text: str, + config: Optional[Dict[str, Any]], + ) -> Optional[WorkerStageSession]: + session_worker_ids: Set[str] = set() + if isinstance(config, dict): + config["_session_worker_ids"] = session_worker_ids + + return cls._start_worker_session( + worker_manager, + worker_type="pipeline", + title="Pipeline run", + description=pipeline_text, + pipe_text=pipeline_text, + config=config, + completion_label="Pipeline completed", + error_label="Pipeline error", + session_worker_ids=session_worker_ids, + ) \ No newline at end of file diff --git a/Store/Folder.py b/Store/Folder.py index 7290f86..fb912eb 100644 --- a/Store/Folder.py +++ b/Store/Folder.py @@ -4,7 +4,7 @@ import json import re import shutil import sys -from fnmatch import fnmatch, translate +from fnmatch import fnmatch from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -177,7 +177,7 @@ class Folder(Store): Checks for sidecars (.metadata, .tag) and imports them before renaming. Also ensures all files have a title: tag. """ - from API.folder import API_folder_store, read_sidecar, write_sidecar, find_sidecar + from API.folder import API_folder_store, read_sidecar, find_sidecar try: with API_folder_store(location_path) as db: diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index 019e83f..c61786a 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -1894,6 +1894,61 @@ class HydrusNetwork(Store): debug(f"{self._log_prefix()} add_url_bulk failed: {exc}") return False + def add_tags_bulk(self, items: List[tuple[str, List[str]]], *, service_name: str | None = None) -> bool: + """Bulk add tags to multiple Hydrus files. + + Groups files by identical tag-sets and uses the Hydrus `mutate_tags_by_key` + call (when a service key is available) to reduce the number of API calls. + Falls back to per-hash `add_tag` calls if necessary. + """ + try: + client = self._client + if client is None: + debug(f"{self._log_prefix()} add_tags_bulk: client unavailable") + return False + + # Group by canonical tag set (sorted tuple) to batch identical additions + buckets: dict[tuple[str, ...], list[str]] = {} + for file_identifier, tags in items or []: + h = str(file_identifier or "").strip().lower() + if len(h) != 64: + continue + tlist = [str(t).strip().lower() for t in (tags or []) if isinstance(t, str) and str(t).strip()] + if not tlist: + continue + key = tuple(sorted(tlist)) + buckets.setdefault(key, []).append(h) + + if not buckets: + return False + + svc = service_name or "my tags" + service_key = self._get_service_key(svc) + any_success = False + + for tag_tuple, hashes in buckets.items(): + try: + if service_key: + # Mutate tags for many hashes in a single request + client.mutate_tags_by_key(hashes=hashes, service_key=service_key, add_tags=list(tag_tuple)) + any_success = True + continue + except Exception as exc: + debug(f"{self._log_prefix()} add_tags_bulk mutate failed for tags {tag_tuple}: {exc}") + + # Fallback: apply per-hash add_tag + for h in hashes: + try: + client.add_tag(h, list(tag_tuple), svc) + any_success = True + except Exception: + continue + + return any_success + except Exception as exc: + debug(f"{self._log_prefix()} add_tags_bulk failed: {exc}") + return False + def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: """Delete one or more url from a Hydrus file.""" try: diff --git a/Store/ZeroTier.py b/Store/ZeroTier.py index 054d862..fdabd36 100644 --- a/Store/ZeroTier.py +++ b/Store/ZeroTier.py @@ -20,9 +20,6 @@ Notes: from __future__ import annotations -import json -import sys -import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -355,7 +352,6 @@ class ZeroTier(Store): Returns the file hash on success, or None on failure. """ - from SYS.utils import sha256_file p = Path(file_path) if not p.exists(): @@ -404,17 +400,60 @@ class ZeroTier(Store): data.append(("url", u)) files = {"file": (p.name, fh, "application/octet-stream")} - resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout) - resp.raise_for_status() - if resp.status_code in (200, 201): + # Prefer `requests` for local testing / WSGI servers which may not accept + # chunked uploads reliably with httpx/httpcore. Fall back to httpx otherwise. + try: try: - payload = resp.json() - file_hash = payload.get("hash") or payload.get("file_hash") - return file_hash - except Exception: + import requests + # Convert data list-of-tuples to dict for requests (acceptable for repeated fields) + data_dict = {} + for k, v in data: + if k in data_dict: + existing = data_dict[k] + if not isinstance(existing, list): + data_dict[k] = [existing] + data_dict[k].append(v) + else: + data_dict[k] = v + r = requests.post(url, headers=headers, files=files, data=data_dict or None, timeout=self._timeout) + if r.status_code in (200, 201): + try: + payload = r.json() + file_hash = payload.get("hash") or payload.get("file_hash") + return file_hash + except Exception: + return None + try: + debug(f"[zerotier-debug] upload failed (requests) status={r.status_code} body={r.text}") + except Exception: + pass + debug(f"ZeroTier add_file failed (requests): status {r.status_code} body={getattr(r, 'text', '')}") return None - debug(f"ZeroTier add_file failed: status {resp.status_code}") - return None + except Exception: + import httpx + resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout) + # Note: some environments may not create request.files correctly; capture body for debugging + try: + if resp.status_code in (200, 201): + try: + payload = resp.json() + file_hash = payload.get("hash") or payload.get("file_hash") + return file_hash + except Exception: + return None + # Debug output to help tests capture server response + try: + debug(f"[zerotier-debug] upload failed status={resp.status_code} body={resp.text}") + except Exception: + pass + debug(f"ZeroTier add_file failed: status {resp.status_code} body={getattr(resp, 'text', '')}") + return None + except Exception as exc: + debug(f"ZeroTier add_file exception: {exc}") + return None + except Exception as exc: + debug(f"ZeroTier add_file exception: {exc}") + return None except Exception as exc: debug(f"ZeroTier add_file exception: {exc}") return None diff --git a/Store/registry.py b/Store/registry.py index 831889b..d58ead1 100644 --- a/Store/registry.py +++ b/Store/registry.py @@ -15,8 +15,7 @@ import importlib import inspect import pkgutil import re -from pathlib import Path -from typing import Any, Dict, Iterable, Optional, Type +from typing import Any, Dict, Optional, Type from SYS.logger import debug from SYS.utils import expand_path diff --git a/TUI.py b/TUI.py index b247283..25eb91f 100644 --- a/TUI.py +++ b/TUI.py @@ -6,7 +6,6 @@ import json import re import sys import subprocess -import asyncio from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple diff --git a/TUI/menu_actions.py b/TUI/menu_actions.py index d6e66db..629700e 100644 --- a/TUI/menu_actions.py +++ b/TUI/menu_actions.py @@ -5,7 +5,7 @@ from __future__ import annotations import sys from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence +from typing import Dict, Iterable, List, Sequence BASE_DIR = Path(__file__).resolve().parent ROOT_DIR = BASE_DIR.parent diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index 4d59b92..d00efad 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -1,12 +1,9 @@ from textual.app import ComposeResult from textual.screen import ModalScreen from textual.containers import Container, Horizontal, Vertical, ScrollableContainer -from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, OptionList, Footer, Select +from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, Select from textual import on, work -from textual.message import Message -from typing import Dict, Any, List, Optional -import os -import json +from typing import Any from pathlib import Path from SYS.config import load_config, save_config, global_config diff --git a/TUI/modalscreen/download.py b/TUI/modalscreen/download.py index 9713feb..6178fee 100644 --- a/TUI/modalscreen/download.py +++ b/TUI/modalscreen/download.py @@ -9,11 +9,10 @@ This modal allows users to specify: from textual.app import ComposeResult from textual.screen import ModalScreen -from textual.containers import Container, Horizontal, Vertical, ScrollableContainer +from textual.containers import Container, Horizontal, Vertical from textual.widgets import ( Static, Button, - Label, Select, Checkbox, TextArea, @@ -448,8 +447,6 @@ class DownloadModal(ModalScreen): try: # Capture output from the cmdlet using temp files (more reliable than redirect) - import tempfile - import subprocess # Try normal redirect first import io @@ -461,7 +458,7 @@ class DownloadModal(ModalScreen): # Always capture output try: with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf): - logger.info(f"Calling download_cmdlet...") + logger.info("Calling download_cmdlet...") cmd_config = ( dict(self.config) if isinstance(self.config, @@ -637,7 +634,7 @@ class DownloadModal(ModalScreen): # Also append detailed error info to worker stdout for visibility if worker: - worker.append_stdout(f"\n❌ DOWNLOAD FAILED\n") + worker.append_stdout("\n❌ DOWNLOAD FAILED\n") worker.append_stdout(f"Reason: {error_reason}\n") if stderr_text and stderr_text.strip(): worker.append_stdout( @@ -1169,7 +1166,7 @@ class DownloadModal(ModalScreen): url.endswith(".pdf") or "pdf" in url.lower() for url in url ) if all_pdfs: - logger.info(f"All url are PDFs - creating pseudo-playlist") + logger.info("All url are PDFs - creating pseudo-playlist") self._handle_pdf_playlist(url) return @@ -1646,7 +1643,7 @@ class DownloadModal(ModalScreen): break if not json_line: - logger.error(f"No JSON found in get-tag output") + logger.error("No JSON found in get-tag output") logger.debug(f"Raw output: {output}") try: self.app.call_from_thread( diff --git a/TUI/modalscreen/export.py b/TUI/modalscreen/export.py index df72dcb..11c9faa 100644 --- a/TUI/modalscreen/export.py +++ b/TUI/modalscreen/export.py @@ -3,20 +3,16 @@ from textual.app import ComposeResult from textual.screen import ModalScreen from textual.containers import Container, Horizontal, Vertical -from textual.widgets import Static, Button, Input, TextArea, Tree, Select +from textual.widgets import Static, Button, Input, TextArea, Select from textual.binding import Binding import logging -from typing import Optional, Any +from typing import Optional from pathlib import Path -import json import sys -import subprocess -from datetime import datetime # Add parent directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent)) from SYS.utils import format_metadata_value -from SYS.config import load_config logger = logging.getLogger(__name__) @@ -147,7 +143,7 @@ class ExportModal(ModalScreen): if not metadata: logger.info( - f"_get_metadata_text - No metadata found, returning 'No metadata available'" + "_get_metadata_text - No metadata found, returning 'No metadata available'" ) return "No metadata available" @@ -184,7 +180,7 @@ class ExportModal(ModalScreen): ) return "\n".join(lines) else: - logger.info(f"_get_metadata_text - No matching fields found in metadata") + logger.info("_get_metadata_text - No matching fields found in metadata") return "No metadata available" def compose(self) -> ComposeResult: diff --git a/TUI/modalscreen/search.py b/TUI/modalscreen/search.py index f1ab2a1..4676e35 100644 --- a/TUI/modalscreen/search.py +++ b/TUI/modalscreen/search.py @@ -2,7 +2,7 @@ from textual.app import ComposeResult from textual.screen import ModalScreen -from textual.containers import Container, Horizontal, Vertical +from textual.containers import Horizontal, Vertical from textual.widgets import Static, Button, Input, Select, DataTable, TextArea from textual.binding import Binding from textual.message import Message @@ -363,7 +363,7 @@ class SearchModal(ModalScreen): tags_text = "\n".join(tags) self.tags_textarea.text = tags_text - logger.info(f"[search-modal] Populated tags textarea from result") + logger.info("[search-modal] Populated tags textarea from result") async def _download_book(self, result: Any) -> None: """Download a book from OpenLibrary using the provider.""" diff --git a/TUI/modalscreen/selection_modal.py b/TUI/modalscreen/selection_modal.py index c1ed3ea..4e3fcc7 100644 --- a/TUI/modalscreen/selection_modal.py +++ b/TUI/modalscreen/selection_modal.py @@ -1,8 +1,8 @@ from textual.app import ComposeResult from textual.screen import ModalScreen from textual.containers import Container, ScrollableContainer -from textual.widgets import Static, Button, Label -from typing import List, Callable +from textual.widgets import Static, Button +from typing import List class SelectionModal(ModalScreen[str]): """A modal for selecting a type from a list of strings.""" diff --git a/TUI/modalscreen/workers.py b/TUI/modalscreen/workers.py index 1bfc5bd..7da9ac9 100644 --- a/TUI/modalscreen/workers.py +++ b/TUI/modalscreen/workers.py @@ -238,7 +238,7 @@ class WorkersModal(ModalScreen): "---", "No workers running" ) - logger.debug(f"[workers-modal] No running workers to display") + logger.debug("[workers-modal] No running workers to display") return logger.debug( @@ -319,7 +319,7 @@ class WorkersModal(ModalScreen): "---", "No finished workers" ) - logger.debug(f"[workers-modal] No finished workers to display") + logger.debug("[workers-modal] No finished workers to display") return logger.info( @@ -399,7 +399,7 @@ class WorkersModal(ModalScreen): workers_list = None if event.control == self.running_table: workers_list = self.running_workers - logger.debug(f"[workers-modal] Highlighted in running table") + logger.debug("[workers-modal] Highlighted in running table") elif event.control == self.finished_table: workers_list = self.finished_workers logger.debug( @@ -442,7 +442,7 @@ class WorkersModal(ModalScreen): workers_list = None if event.data_table == self.running_table: workers_list = self.running_workers - logger.debug(f"[workers-modal] Cell highlighted in running table") + logger.debug("[workers-modal] Cell highlighted in running table") elif event.data_table == self.finished_table: workers_list = self.finished_workers logger.debug( @@ -502,7 +502,7 @@ class WorkersModal(ModalScreen): self.stdout_display.cursor_location = (len(combined_text) - 1, 0) except Exception: pass - logger.info(f"[workers-modal] Updated stdout display successfully") + logger.info("[workers-modal] Updated stdout display successfully") except Exception as e: logger.error( f"[workers-modal] Error updating stdout display: {e}", diff --git a/TUI/pipeline_runner.py b/TUI/pipeline_runner.py index f1b182d..414635e 100644 --- a/TUI/pipeline_runner.py +++ b/TUI/pipeline_runner.py @@ -22,13 +22,9 @@ for path in (ROOT_DIR, BASE_DIR): sys.path.insert(0, str_path) from SYS import pipeline as ctx -# Lazily import CLI dependencies to avoid import-time failures in test environments -try: - from CLI import ConfigLoader, PipelineExecutor as CLIPipelineExecutor, WorkerManagerRegistry -except Exception: - ConfigLoader = None - CLIPipelineExecutor = None - WorkerManagerRegistry = None +from CLI import ConfigLoader +from SYS.pipeline import PipelineExecutor +from SYS.worker import WorkerManagerRegistry from SYS.logger import set_debug from SYS.rich_display import capture_rich_output from SYS.result_table import Table @@ -89,7 +85,7 @@ class PipelineRunner: if executor is not None: self._executor = executor else: - self._executor = CLIPipelineExecutor(config_loader=self._config_loader) if CLIPipelineExecutor else None + self._executor = PipelineExecutor(config_loader=self._config_loader) self._worker_manager = None @property diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 7988c5f..6e64694 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -499,6 +499,9 @@ class Add_File(Cmdlet): pending_url_associations: Dict[str, List[tuple[str, List[str]]]] = {} + pending_tag_associations: Dict[str, + List[tuple[str, + List[str]]]] = {} successes = 0 failures = 0 @@ -612,6 +615,8 @@ class Add_File(Cmdlet): collect_relationship_pairs=pending_relationship_pairs, defer_url_association=defer_url_association, pending_url_associations=pending_url_associations, + defer_tag_association=defer_url_association, + pending_tag_associations=pending_tag_associations, suppress_last_stage_overlay=want_final_search_file, auto_search_file=auto_search_file_after_add, store_instance=storage_registry, @@ -664,6 +669,17 @@ class Add_File(Cmdlet): except Exception: pass + # Apply deferred tag associations (bulk) if collected + if pending_tag_associations: + try: + Add_File._apply_pending_tag_associations( + pending_tag_associations, + config, + store_instance=storage_registry + ) + except Exception: + pass + # Always end add-file -store (when last stage) by showing item detail panels. # Legacy search-file refresh is no longer used for final display. if want_final_search_file and collected_payloads: @@ -1854,6 +1870,10 @@ class Add_File(Cmdlet): pending_url_associations: Optional[Dict[str, List[tuple[str, List[str]]]]] = None, + defer_tag_association: bool = False, + pending_tag_associations: Optional[Dict[str, + List[tuple[str, + List[str]]]]] = None, suppress_last_stage_overlay: bool = False, auto_search_file: bool = True, store_instance: Optional[Store] = None, @@ -2072,15 +2092,22 @@ class Add_File(Cmdlet): resolved_hash = chosen_hash if hydrus_like_backend and tags: - try: - adder = getattr(backend, "add_tag", None) - if callable(adder): - debug( - f"[add-file] Applying {len(tags)} tag(s) post-upload to Hydrus" - ) - adder(resolved_hash, list(tags)) - except Exception as exc: - log(f"[add-file] Hydrus post-upload tagging failed: {exc}", file=sys.stderr) + # Support deferring tag application for batching bulk operations + if defer_tag_association and pending_tag_associations is not None: + try: + pending_tag_associations.setdefault(str(backend_name), []).append((str(resolved_hash), list(tags))) + except Exception: + pass + else: + try: + adder = getattr(backend, "add_tag", None) + if callable(adder): + debug( + f"[add-file] Applying {len(tags)} tag(s) post-upload to Hydrus" + ) + adder(resolved_hash, list(tags)) + except Exception as exc: + log(f"[add-file] Hydrus post-upload tagging failed: {exc}", file=sys.stderr) # If we have url(s), ensure they get associated with the destination file. # This mirrors `add-url` behavior but avoids emitting extra pipeline noise. @@ -2322,6 +2349,47 @@ class Add_File(Cmdlet): except Exception: continue + @staticmethod + def _apply_pending_tag_associations( + pending: Dict[str, + List[tuple[str, + List[str]]]], + config: Dict[str, + Any], + store_instance: Optional[Store] = None, + ) -> None: + """Apply deferred tag associations in bulk, grouped per backend.""" + + try: + store = store_instance if store_instance is not None else Store(config) + except Exception: + return + + for backend_name, pairs in (pending or {}).items(): + if not pairs: + continue + try: + backend = store[backend_name] + except Exception: + continue + + # Try bulk variant first + bulk = getattr(backend, "add_tags_bulk", None) + if callable(bulk): + try: + bulk([(h, t) for h, t in pairs]) + continue + except Exception: + pass + + single = getattr(backend, "add_tag", None) + if callable(single): + for h, t in pairs: + try: + single(h, t) + except Exception: + continue + @staticmethod def _load_sidecar_bundle( media_path: Path, diff --git a/cmdlet/add_relationship.py b/cmdlet/add_relationship.py index e6195de..7d6cbea 100644 --- a/cmdlet/add_relationship.py +++ b/cmdlet/add_relationship.py @@ -1097,7 +1097,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: ] if not relationship_tags: - log(f"No relationship tags found in sidecar", file=sys.stderr) + log("No relationship tags found in sidecar", file=sys.stderr) return 0 # Not an error, just nothing to do # Get the file hash from result (should have been set by add-file) @@ -1166,7 +1166,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: ) return 0 elif error_count == 0: - log(f"No relationships to set", file=sys.stderr) + log("No relationships to set", file=sys.stderr) return 0 # Success with nothing to do else: log(f"Failed with {error_count} error(s)", file=sys.stderr) diff --git a/cmdlet/add_url.py b/cmdlet/add_url.py index 0bcf90a..d437063 100644 --- a/cmdlet/add_url.py +++ b/cmdlet/add_url.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Sequence, Tuple import sys from SYS import pipeline as ctx diff --git a/cmdlet/delete_file.py b/cmdlet/delete_file.py index 55f958d..e24643f 100644 --- a/cmdlet/delete_file.py +++ b/cmdlet/delete_file.py @@ -7,7 +7,6 @@ import sys from pathlib import Path from SYS.logger import debug, log -from SYS.utils import format_bytes from Store.Folder import Folder from Store import Store from . import _shared as sh diff --git a/cmdlet/delete_tag.py b/cmdlet/delete_tag.py index f92434f..8fa810c 100644 --- a/cmdlet/delete_tag.py +++ b/cmdlet/delete_tag.py @@ -2,10 +2,8 @@ from __future__ import annotations from typing import Any, Dict, Sequence from pathlib import Path -import json import sys -from SYS import models from SYS import pipeline as ctx from . import _shared as sh diff --git a/cmdlet/delete_url.py b/cmdlet/delete_url.py index b9b1fe9..f506f3c 100644 --- a/cmdlet/delete_url.py +++ b/cmdlet/delete_url.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Sequence, Tuple import sys from SYS import pipeline as ctx diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index 55c0c10..e5d0009 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -15,7 +15,6 @@ from typing import Any, Dict, List, Optional, Sequence from urllib.parse import urlparse from contextlib import AbstractContextManager, nullcontext -import requests from API.HTTP import _download_direct_file from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult @@ -26,7 +25,6 @@ from SYS.rich_display import stderr_console as get_stderr_console from SYS import pipeline as pipeline_context from SYS.utils import sha256_file from SYS.metadata import normalize_urls as normalize_url_list -from rich.prompt import Confirm from tool.ytdlp import ( YtDlpTool, @@ -948,7 +946,7 @@ class Download_File(Cmdlet): from Store import Store from API.HydrusNetwork import is_hydrus_available - debug(f"[download-file] Initializing storage interface...") + debug("[download-file] Initializing storage interface...") storage = Store(config=config or {}, suppress_debug=True) hydrus_available = bool(is_hydrus_available(config or {})) @@ -1338,7 +1336,7 @@ class Download_File(Cmdlet): table.set_source_command("download-file", [url]) debug(f"[ytdlp.formatlist] Displaying format selection table for {url}") - debug(f"[ytdlp.formatlist] Provider: ytdlp (routing to download-file via TABLE_AUTO_STAGES)") + debug("[ytdlp.formatlist] Provider: ytdlp (routing to download-file via TABLE_AUTO_STAGES)") results_list: List[Dict[str, Any]] = [] for idx, fmt in enumerate(filtered_formats, 1): @@ -1420,7 +1418,7 @@ class Download_File(Cmdlet): f"[ytdlp.formatlist] When user selects @N, will invoke: download-file {url} -query 'format:'" ) - log(f"", file=sys.stderr) + log("", file=sys.stderr) return 0 return None @@ -2054,7 +2052,7 @@ class Download_File(Cmdlet): forced_single_format_id = None forced_single_format_for_batch = False - debug(f"[download-file] Checking if format table should be shown...") + debug("[download-file] Checking if format table should be shown...") early_ret = self._maybe_show_format_table_for_single_url( mode=mode, clip_spec=clip_spec, @@ -2763,7 +2761,7 @@ class Download_File(Cmdlet): debug(f"[download-file] Processing {total_selection} selected item(s) from table...") for idx, run_args in enumerate(selection_runs, 1): debug(f"[download-file] Item {idx}/{total_selection}: {run_args}") - debug(f"[download-file] Re-invoking download-file for selected item...") + debug("[download-file] Re-invoking download-file for selected item...") exit_code = self._run_impl(None, run_args, config) if exit_code == 0: successes += 1 diff --git a/cmdlet/get_file.py b/cmdlet/get_file.py index 16ac77f..b9ba2eb 100644 --- a/cmdlet/get_file.py +++ b/cmdlet/get_file.py @@ -92,7 +92,7 @@ class Get_File(sh.Cmdlet): debug(f"[get-file] Backend retrieved: {type(backend).__name__}") # Get file metadata to determine name and extension - debug(f"[get-file] Getting metadata for hash...") + debug("[get-file] Getting metadata for hash...") metadata = backend.get_metadata(file_hash) if not metadata: log(f"Error: File metadata not found for hash {file_hash}") @@ -228,7 +228,7 @@ class Get_File(sh.Cmdlet): } ) - debug(f"[get-file] Completed successfully") + debug("[get-file] Completed successfully") return 0 def _open_file_default(self, path: Path) -> None: diff --git a/cmdlet/get_metadata.py b/cmdlet/get_metadata.py index 073d669..816e2d5 100644 --- a/cmdlet/get_metadata.py +++ b/cmdlet/get_metadata.py @@ -5,7 +5,6 @@ import json import sys from SYS.logger import log -from pathlib import Path from . import _shared as sh diff --git a/cmdlet/get_note.py b/cmdlet/get_note.py index bbf98ba..5e1b6b2 100644 --- a/cmdlet/get_note.py +++ b/cmdlet/get_note.py @@ -7,7 +7,6 @@ import sys from SYS.logger import log from SYS import pipeline as ctx -from SYS.result_table import Table from . import _shared as sh Cmdlet = sh.Cmdlet diff --git a/cmdlet/get_relationship.py b/cmdlet/get_relationship.py index a5adee0..5ab0e04 100644 --- a/cmdlet/get_relationship.py +++ b/cmdlet/get_relationship.py @@ -1,13 +1,11 @@ from __future__ import annotations -from typing import Any, Dict, Sequence, List, Optional -import json +from typing import Any, Dict, Sequence, Optional import sys from pathlib import Path from SYS.logger import log -from SYS import models from SYS import pipeline as ctx from API import HydrusNetwork as hydrus_wrapper from . import _shared as sh @@ -22,8 +20,6 @@ fetch_hydrus_metadata = sh.fetch_hydrus_metadata should_show_help = sh.should_show_help get_field = sh.get_field from API.folder import API_folder_store -from SYS.config import get_local_storage_path -from SYS.result_table import Table from Store import Store CMDLET = Cmdlet( @@ -512,7 +508,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if source_title and source_title != "Unknown": metadata["Title"] = source_title - table = ItemDetailView(f"Relationships", item_metadata=metadata + table = ItemDetailView("Relationships", item_metadata=metadata ).init_command("get-relationship", []) diff --git a/cmdlet/get_tag.py b/cmdlet/get_tag.py index 72a42ab..bfd8264 100644 --- a/cmdlet/get_tag.py +++ b/cmdlet/get_tag.py @@ -25,8 +25,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple from SYS import pipeline as ctx -from API import HydrusNetwork -from API.folder import read_sidecar, write_sidecar, find_sidecar, API_folder_store +from API.folder import read_sidecar, write_sidecar from . import _shared as sh normalize_hash = sh.normalize_hash @@ -36,7 +35,6 @@ CmdletArg = sh.CmdletArg SharedArgs = sh.SharedArgs parse_cmdlet_args = sh.parse_cmdlet_args get_field = sh.get_field -from SYS.config import get_local_storage_path try: from SYS.metadata import extract_title @@ -944,7 +942,7 @@ def _scrape_url_metadata( ) except json_module.JSONDecodeError: pass - except Exception as e: + except Exception: pass # Silently ignore if we can't get playlist entries # Fallback: if still no tags detected, get from first item diff --git a/cmdlet/merge_file.py b/cmdlet/merge_file.py index 48aa577..d5a1b23 100644 --- a/cmdlet/merge_file.py +++ b/cmdlet/merge_file.py @@ -320,7 +320,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: f"Mixed file types detected: {', '.join(sorted(file_types))}", file=sys.stderr ) - log(f"Can only merge files of the same type", file=sys.stderr) + log("Can only merge files of the same type", file=sys.stderr) return 1 file_kind = list(file_types)[0] if file_types else "other" @@ -524,7 +524,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: current_time_ms = 0 log(f"Analyzing {len(files)} files for chapter information...", file=sys.stderr) - logger.info(f"[merge-file] Analyzing files for chapters") + logger.info("[merge-file] Analyzing files for chapters") for file_path in files: # Get duration using ffprobe @@ -767,14 +767,14 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: logger.exception(f"[merge-file] ffmpeg process error: {e}") raise - log(f"Merge successful, adding chapters metadata...", file=sys.stderr) + log("Merge successful, adding chapters metadata...", file=sys.stderr) # Step 5: Embed chapters into container (MKA, MP4/M4A, or note limitation) if output_format == "mka" or output.suffix.lower() == ".mka": # MKA/MKV format has native chapter support via FFMetadata # Re-mux the file with chapters embedded (copy streams, no re-encode) - log(f"Embedding chapters into Matroska container...", file=sys.stderr) - logger.info(f"[merge-file] Adding chapters to MKA file via FFMetadata") + log("Embedding chapters into Matroska container...", file=sys.stderr) + logger.info("[merge-file] Adding chapters to MKA file via FFMetadata") temp_output = output.parent / f".temp_{output.stem}.mka" @@ -783,7 +783,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: if mkvmerge_path: # mkvmerge is the best tool for embedding chapters in Matroska files - log(f"Using mkvmerge for optimal chapter embedding...", file=sys.stderr) + log("Using mkvmerge for optimal chapter embedding...", file=sys.stderr) cmd2 = [ mkvmerge_path, "-o", @@ -795,7 +795,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: else: # Fallback to ffmpeg with proper chapter embedding for Matroska log( - f"Using ffmpeg for chapter embedding (install mkvtoolnix for better quality)...", + "Using ffmpeg for chapter embedding (install mkvtoolnix for better quality)...", file=sys.stderr, ) # For Matroska files, the metadata must be provided via -f ffmetadata input @@ -838,12 +838,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: if output.exists(): output.unlink() shutil.move(str(temp_output), str(output)) - log(f"✓ Chapters successfully embedded!", file=sys.stderr) - logger.info(f"[merge-file] Chapters embedded successfully") + log("✓ Chapters successfully embedded!", file=sys.stderr) + logger.info("[merge-file] Chapters embedded successfully") except Exception as e: logger.warning(f"[merge-file] Could not replace file: {e}") log( - f"Warning: Could not embed chapters, using merge without chapters", + "Warning: Could not embed chapters, using merge without chapters", file=sys.stderr, ) try: @@ -852,12 +852,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: pass else: logger.warning( - f"[merge-file] Chapter embedding did not create output" + "[merge-file] Chapter embedding did not create output" ) except Exception as e: logger.exception(f"[merge-file] Chapter embedding failed: {e}") log( - f"Warning: Chapter embedding failed, using merge without chapters", + "Warning: Chapter embedding failed, using merge without chapters", file=sys.stderr, ) elif output_format in {"m4a", @@ -865,15 +865,15 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: ".m4b", ".mp4"]: # MP4/M4A format has native chapter support via iTunes metadata atoms - log(f"Embedding chapters into MP4 container...", file=sys.stderr) + log("Embedding chapters into MP4 container...", file=sys.stderr) logger.info( - f"[merge-file] Adding chapters to M4A/MP4 file via iTunes metadata" + "[merge-file] Adding chapters to M4A/MP4 file via iTunes metadata" ) temp_output = output.parent / f".temp_{output.stem}{output.suffix}" # ffmpeg embeds chapters in MP4 using -map_metadata and -map_chapters - log(f"Using ffmpeg for MP4 chapter embedding...", file=sys.stderr) + log("Using ffmpeg for MP4 chapter embedding...", file=sys.stderr) cmd2 = [ ffmpeg_path, "-y", @@ -916,14 +916,14 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: output.unlink() shutil.move(str(temp_output), str(output)) log( - f"✓ Chapters successfully embedded in MP4!", + "✓ Chapters successfully embedded in MP4!", file=sys.stderr ) - logger.info(f"[merge-file] MP4 chapters embedded successfully") + logger.info("[merge-file] MP4 chapters embedded successfully") except Exception as e: logger.warning(f"[merge-file] Could not replace file: {e}") log( - f"Warning: Could not embed chapters, using merge without chapters", + "Warning: Could not embed chapters, using merge without chapters", file=sys.stderr, ) try: @@ -932,12 +932,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: pass else: logger.warning( - f"[merge-file] MP4 chapter embedding did not create output" + "[merge-file] MP4 chapter embedding did not create output" ) except Exception as e: logger.exception(f"[merge-file] MP4 chapter embedding failed: {e}") log( - f"Warning: MP4 chapter embedding failed, using merge without chapters", + "Warning: MP4 chapter embedding failed, using merge without chapters", file=sys.stderr, ) else: @@ -945,7 +945,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool: logger.info( f"[merge-file] Format {output_format} does not have native chapter support" ) - log(f"Note: For chapter support, use MKA or M4A format", file=sys.stderr) + log("Note: For chapter support, use MKA or M4A format", file=sys.stderr) # Clean up temp files try: diff --git a/cmdlet/provider_table.py b/cmdlet/provider_table.py index 68297b6..6c3435a 100644 --- a/cmdlet/provider_table.py +++ b/cmdlet/provider_table.py @@ -4,7 +4,7 @@ import sys from typing import Any, Dict, Iterable, Sequence from . import _shared as sh -from SYS.logger import log, debug +from SYS.logger import log from SYS import pipeline as ctx from SYS.result_table_adapters import get_provider @@ -43,7 +43,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: try: provider = get_provider(provider_name) - except Exception as exc: + except Exception: log(f"Unknown provider: {provider_name}", file=sys.stderr) return 1 diff --git a/cmdlet/screen_shot.py b/cmdlet/screen_shot.py index 76a1879..d908701 100644 --- a/cmdlet/screen_shot.py +++ b/cmdlet/screen_shot.py @@ -656,7 +656,7 @@ def _capture( # Attempt platform-specific target capture if requested (and not PDF) element_captured = False if options.prefer_platform_target and format_name != "pdf": - debug(f"[_capture] Target capture enabled") + debug("[_capture] Target capture enabled") debug("Attempting platform-specific content capture...") progress.step("capturing locating target") try: @@ -913,7 +913,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: url_to_process.append((str(url), item)) if not url_to_process: - log(f"No url to process for screen-shot cmdlet", file=sys.stderr) + log("No url to process for screen-shot cmdlet", file=sys.stderr) return 1 debug(f"[_run] url to process: {[u for u, _ in url_to_process]}") @@ -1157,7 +1157,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: progress.close_local_ui(force_complete=True) if not all_emitted: - log(f"No screenshots were successfully captured", file=sys.stderr) + log("No screenshots were successfully captured", file=sys.stderr) return 1 # Log completion message (keep this as normal output) diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index 954a31c..2e5bcf8 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import Any, Dict, Sequence, List, Optional -import importlib import uuid from pathlib import Path import re diff --git a/cmdnat/adjective.py b/cmdnat/adjective.py index aee2756..c9d5b49 100644 --- a/cmdnat/adjective.py +++ b/cmdnat/adjective.py @@ -1,8 +1,8 @@ import json import os import sys -from typing import List, Dict, Any, Optional, Sequence -from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args +from typing import List, Dict, Any, Sequence +from cmdlet._shared import Cmdlet, CmdletArg from SYS.logger import log from SYS.result_table import Table from SYS import pipeline as ctx diff --git a/cmdnat/config.py b/cmdnat/config.py index 63ed51d..f23bc66 100644 --- a/cmdnat/config.py +++ b/cmdnat/config.py @@ -213,7 +213,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: # Check if we're in an interactive terminal and can launch a Textual modal if sys.stdin.isatty() and not piped_result: try: - from textual.app import App, ComposeResult + from textual.app import App from TUI.modalscreen.config_modal import ConfigModal class ConfigApp(App): diff --git a/cmdnat/pipe.py b/cmdnat/pipe.py index 3332f50..6dcb771 100644 --- a/cmdnat/pipe.py +++ b/cmdnat/pipe.py @@ -4,7 +4,6 @@ import sys import json import socket import re -import subprocess from urllib.parse import urlparse, parse_qs from pathlib import Path from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args, resolve_tidal_manifest_path diff --git a/cmdnat/status.py b/cmdnat/status.py index 76d7995..7f500f0 100644 --- a/cmdnat/status.py +++ b/cmdnat/status.py @@ -1,15 +1,12 @@ from __future__ import annotations -import sys import shutil -from typing import Any, Dict, List, Optional, Sequence, Tuple -from datetime import datetime +from typing import Any, Dict, List -from cmdlet._shared import Cmdlet, CmdletArg +from cmdlet._shared import Cmdlet from SYS import pipeline as ctx from SYS.result_table import Table -from SYS.logger import log, set_debug, debug -from SYS.rich_display import stdout_console +from SYS.logger import set_debug, debug CMDLET = Cmdlet( name=".status", diff --git a/cmdnat/zerotier.py b/cmdnat/zerotier.py index ab9f55d..a26b57c 100644 --- a/cmdnat/zerotier.py +++ b/cmdnat/zerotier.py @@ -1,15 +1,14 @@ -import os import sys import requests from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, Sequence # Add project root to sys.path root = Path(__file__).resolve().parent.parent if str(root) not in sys.path: sys.path.insert(0, str(root)) -from cmdlet._shared import Cmdlet, CmdletArg +from cmdlet._shared import Cmdlet from SYS.config import load_config from SYS.result_table import Table from API import zerotier as zt diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index 186833f..06351e5 100644 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -203,7 +203,6 @@ def run_platform_bootstrap(repo_root: Path) -> int: def playwright_package_installed() -> bool: try: - import playwright # type: ignore return True except Exception: @@ -751,7 +750,7 @@ def main() -> int: user_bin = Path(os.environ.get("USERPROFILE", str(home))) / "bin" mm_bat = user_bin / "mm.bat" - print(f"Checking for shim files:") + print("Checking for shim files:") print(f" mm.bat: {'✓' if mm_bat.exists() else '✗'} ({mm_bat})") print() @@ -760,14 +759,14 @@ def main() -> int: if "REPO=" in bat_content or "ENTRY=" in bat_content: print(f" mm.bat content looks valid ({len(bat_content)} bytes)") else: - print(f" ⚠️ mm.bat content may be corrupted") + print(" ⚠️ mm.bat content may be corrupted") print() # Check PATH path = os.environ.get("PATH", "") user_bin_str = str(user_bin) in_path = user_bin_str in path - print(f"Checking PATH environment variable:") + print("Checking PATH environment variable:") print(f" {user_bin_str} in current session PATH: {'✓' if in_path else '✗'}") # Check registry @@ -792,7 +791,7 @@ def main() -> int: try: result = subprocess.run(["mm", "--help"], capture_output=True, text=True, timeout=5) if result.returncode == 0: - print(f" ✓ 'mm --help' works!") + print(" ✓ 'mm --help' works!") print(f" Output (first line): {result.stdout.split(chr(10))[0]}") else: print(f" ✗ 'mm --help' failed with exit code {result.returncode}") @@ -800,8 +799,8 @@ def main() -> int: print(f" Error: {result.stderr.strip()}") except FileNotFoundError: # mm not found via PATH, try calling the .ps1 directly - print(f" ✗ 'mm' command not found in PATH") - print(f" Shims exist but command is not accessible via PATH") + print(" ✗ 'mm' command not found in PATH") + print(" Shims exist but command is not accessible via PATH") print() print("Attempting to call shim directly...") try: @@ -810,23 +809,23 @@ def main() -> int: capture_output=True, text=True, timeout=5 ) if result.returncode == 0: - print(f" ✓ Direct shim call works!") - print(f" The shim files are valid and functional.") + print(" ✓ Direct shim call works!") + print(" The shim files are valid and functional.") print() print("⚠️ 'mm' is not in PATH, but the shims are working correctly.") print() print("Possible causes and fixes:") - print(f" 1. Terminal needs restart: Close and reopen your terminal/PowerShell") - print(f" 2. PATH reload: Run: $env:Path = [Environment]::GetEnvironmentVariable('PATH', 'User') + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine')") + print(" 1. Terminal needs restart: Close and reopen your terminal/PowerShell") + print(" 2. PATH reload: Run: $env:Path = [Environment]::GetEnvironmentVariable('PATH', 'User') + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine')") print(f" 3. Manual PATH: Add {user_bin} to your system PATH manually") else: - print(f" ✗ Direct shim call failed") + print(" ✗ Direct shim call failed") if result.stderr: print(f" Error: {result.stderr.strip()}") except Exception as e: print(f" ✗ Could not test direct shim: {e}") except subprocess.TimeoutExpired: - print(f" ✗ 'mm' command timed out") + print(" ✗ 'mm' command timed out") except Exception as e: print(f" ✗ Error testing 'mm': {e}") else: @@ -835,7 +834,7 @@ def main() -> int: locations = [home / ".local" / "bin" / "mm", Path("/usr/local/bin/mm"), Path("/usr/bin/mm")] found_shims = [p for p in locations if p.exists()] - print(f"Checking for shim files:") + print("Checking for shim files:") for p in locations: if p.exists(): print(f" mm: ✓ ({p})") @@ -844,23 +843,23 @@ def main() -> int: print(f" mm: ✗ ({p})") if not found_shims: - print(f" mm: ✗ (No shim found in standard locations)") + print(" mm: ✗ (No shim found in standard locations)") print() path = os.environ.get("PATH", "") # Find which 'mm' is actually being run actual_mm = shutil.which("mm") - print(f"Checking PATH environment variable:") + print("Checking PATH environment variable:") if actual_mm: print(f" 'mm' resolved to: {actual_mm}") # Check if it's in a directory on the PATH if any(str(Path(actual_mm).parent) in p for p in path.split(os.pathsep)): - print(f" Command is accessible via current session PATH: ✓") + print(" Command is accessible via current session PATH: ✓") else: - print(f" Command is found but directory may not be in current PATH: ⚠️") + print(" Command is found but directory may not be in current PATH: ⚠️") else: - print(f" 'mm' not found in current session PATH: ✗") + print(" 'mm' not found in current session PATH: ✗") print() # Test if mm command works @@ -868,14 +867,14 @@ def main() -> int: try: result = subprocess.run(["mm", "--help"], capture_output=True, text=True, timeout=5) if result.returncode == 0: - print(f" ✓ 'mm --help' works!") + print(" ✓ 'mm --help' works!") print(f" Output (first line): {result.stdout.split(chr(10))[0]}") else: print(f" ✗ 'mm --help' failed with exit code {result.returncode}") if result.stderr: print(f" Error: {result.stderr.strip()}") except FileNotFoundError: - print(f" ✗ 'mm' command not found in PATH") + print(" ✗ 'mm' command not found in PATH") except Exception as e: print(f" ✗ Error testing 'mm': {e}") @@ -1002,7 +1001,7 @@ def main() -> int: try: _run_cmd([str(python_path), "-m", "ensurepip", "--upgrade"]) - except subprocess.CalledProcessError as exc: + except subprocess.CalledProcessError: print( "Failed to install pip inside the local virtualenv via ensurepip; ensure your Python build includes ensurepip and retry.", file=sys.stderr, @@ -1326,10 +1325,10 @@ if (Test-Path (Join-Path $repo 'CLI.py')) { if not args.quiet: print(f"Installed global launcher to: {user_bin}") - print(f"✓ mm.bat (Command Prompt and PowerShell)") + print("✓ mm.bat (Command Prompt and PowerShell)") print() print("You can now run 'mm' from any terminal window.") - print(f"If 'mm' is not found, restart your terminal or reload PATH:") + print("If 'mm' is not found, restart your terminal or reload PATH:") print(" PowerShell: $env:PATH = [Environment]::GetEnvironmentVariable('PATH','User') + ';' + [Environment]::GetEnvironmentVariable('PATH','Machine')") print(" CMD: path %PATH%") diff --git a/scripts/check_imports.py b/scripts/check_imports.py index 3505f41..5c6dad6 100644 --- a/scripts/check_imports.py +++ b/scripts/check_imports.py @@ -5,6 +5,6 @@ import traceback try: importlib.import_module("CLI") print("CLI imported OK") -except Exception as e: +except Exception: traceback.print_exc() sys.exit(1) diff --git a/scripts/check_pattern.py b/scripts/check_pattern.py new file mode 100644 index 0000000..dd3849c --- /dev/null +++ b/scripts/check_pattern.py @@ -0,0 +1,19 @@ +import re +from pathlib import Path +p = Path(r'c:\Forgejo\Medios-Macina\CLI.py') +s = p.read_text(encoding='utf-8') +pattern = re.compile(r'(?s)if False:\s*class _OldPipelineExecutor:.*?from rich\\.markdown import Markdown\\s*') +m = pattern.search(s) +print('found', bool(m)) +if m: + print('start', m.start(), 'end', m.end()) + print('snippet:', s[m.start():m.start()+120]) +else: + # print a slice around the if False for debugging + i = s.find('if False:') + print('if False index', i) + print('around if False:', s[max(0,i-50):i+200]) + j = s.find('from rich.markdown import Markdown', i) + print('next from rich index after if False', j) + if j!=-1: + print('around that:', s[j-50:j+80]) diff --git a/scripts/check_try_balance.py b/scripts/check_try_balance.py new file mode 100644 index 0000000..c5751ab --- /dev/null +++ b/scripts/check_try_balance.py @@ -0,0 +1,35 @@ +from pathlib import Path +p=Path('SYS/pipeline.py') +s=p.read_text(encoding='utf-8') +lines=s.splitlines() +stack=[] +for i,l in enumerate(lines,1): + stripped=l.strip() + # Skip commented lines + if stripped.startswith('#'): + continue + # compute indent as leading spaces (tabs are converted) + indent = len(l) - len(l.lstrip(' ')) + if stripped.startswith('try:'): + stack.append((indent, i)) + if stripped.startswith('except ') or stripped=='except:' or stripped.startswith('finally:'): + # find the most recent try with same indent + for idx in range(len(stack)-1, -1, -1): + if stack[idx][0] == indent: + stack.pop(idx) + break + else: + # no matching try at same indent + print(f"Found {stripped.split()[0]} at line {i} with no matching try at same indent") + +print('Unmatched try count', len(stack)) +if stack: + print('Unmatched try positions (indent, line):', stack) + for indent, lineno in stack: + start = max(1, lineno - 10) + end = min(len(lines), lineno + 10) + print(f"Context around line {lineno}:") + for i in range(start, end + 1): + print(f"{i:5d}: {lines[i-1]}") +else: + print("All try statements appear matched") diff --git a/scripts/debug_import_vimm.py b/scripts/debug_import_vimm.py index 809178d..f954042 100644 --- a/scripts/debug_import_vimm.py +++ b/scripts/debug_import_vimm.py @@ -1,4 +1,5 @@ -import importlib, traceback +import importlib +import traceback try: m = importlib.import_module('Provider.vimm') diff --git a/scripts/hydrusnetwork.py b/scripts/hydrusnetwork.py index 6870deb..fa6421f 100644 --- a/scripts/hydrusnetwork.py +++ b/scripts/hydrusnetwork.py @@ -28,7 +28,6 @@ import sys import tempfile import urllib.request import zipfile -import shlex import re from pathlib import Path from typing import Optional, Tuple @@ -870,7 +869,7 @@ def main(argv: Optional[list[str]] = None) -> int: args.root = str(default_root) # Ask for destination folder name - dest_input = input(f"Enter folder name for Hydrus [default: hydrusnetwork]: ").strip() + dest_input = input("Enter folder name for Hydrus [default: hydrusnetwork]: ").strip() if dest_input: args.dest_name = dest_input except (EOFError, KeyboardInterrupt): diff --git a/scripts/remote_storage_server.py b/scripts/remote_storage_server.py index c852c68..32a1e05 100644 --- a/scripts/remote_storage_server.py +++ b/scripts/remote_storage_server.py @@ -41,7 +41,6 @@ from __future__ import annotations import os import sys -import json import argparse import logging import threading @@ -54,7 +53,6 @@ from functools import wraps # Add parent directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent)) -from SYS.logger import log # ============================================================================ # CONFIGURATION @@ -419,29 +417,32 @@ def create_app(): filename = sanitize_filename(file_storage.filename or "upload") incoming_dir = STORAGE_PATH / "incoming" - ensure_directory(incoming_dir) target_path = incoming_dir / filename target_path = unique_path(target_path) try: - # Save uploaded file to storage - file_storage.save(str(target_path)) - - # Extract optional metadata - tags = [] - if 'tag' in request.form: - # Support repeated form fields or comma-separated list - tags = request.form.getlist('tag') or [] - if not tags and request.form.get('tag'): - tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()] - - urls = [] - if 'url' in request.form: - urls = request.form.getlist('url') or [] - if not urls and request.form.get('url'): - urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()] - + # Initialize the DB first (run safety checks) before creating any files. with API_folder_store(STORAGE_PATH) as db: + # Ensure the incoming directory exists only after DB safety checks pass. + ensure_directory(incoming_dir) + + # Save uploaded file to storage + file_storage.save(str(target_path)) + + # Extract optional metadata + tags = [] + if 'tag' in request.form: + # Support repeated form fields or comma-separated list + tags = request.form.getlist('tag') or [] + if not tags and request.form.get('tag'): + tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()] + + urls = [] + if 'url' in request.form: + urls = request.form.getlist('url') or [] + if not urls and request.form.get('url'): + urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()] + db.get_or_create_file_entry(target_path) if tags: @@ -723,7 +724,7 @@ def main(): local_ip = "127.0.0.1" print(f"\n{'='*70}") - print(f"Remote Storage Server - Medios-Macina") + print("Remote Storage Server - Medios-Macina") print(f"{'='*70}") print(f"Storage Path: {STORAGE_PATH}") print(f"Local IP: {local_ip}") diff --git a/scripts/remove_old_executor.py b/scripts/remove_old_executor.py new file mode 100644 index 0000000..90db6eb --- /dev/null +++ b/scripts/remove_old_executor.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from pathlib import Path +p = Path(r"c:\Forgejo\Medios-Macina\CLI.py") +s = p.read_text(encoding='utf-8') +start = s.find('\nif False:') +if start == -1: + print('No if False found') +else: + after = s[start+1:] + idx = after.find('\nfrom rich.markdown import Markdown') + if idx == -1: + print('No subsequent import found') + else: + before = s[:start] + rest = after[idx+1:] + new = before + '\nfrom rich.markdown import Markdown\n' + rest + p.write_text(new, encoding='utf-8') + print('Removed legacy block') diff --git a/scripts/zerotier_setup.py b/scripts/zerotier_setup.py index b2b840e..c88b42d 100644 --- a/scripts/zerotier_setup.py +++ b/scripts/zerotier_setup.py @@ -14,10 +14,9 @@ from __future__ import annotations import argparse import json import sys -from typing import Any from pathlib import Path -from SYS.logger import log, debug +from SYS.logger import log try: from API import zerotier diff --git a/tool/florencevision.py b/tool/florencevision.py index ea2aa68..08397da 100644 --- a/tool/florencevision.py +++ b/tool/florencevision.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple from SYS.logger import debug diff --git a/tool/ytdlp.py b/tool/ytdlp.py index 77025ce..4e0f538 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -11,7 +11,6 @@ import sys import threading import time import traceback -from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Sequence, cast