updating and refactoring codebase for improved performance and maintainability

This commit is contained in:
2026-05-03 17:29:32 -07:00
parent b7d3dc5f2d
commit 77cab1bd27
17 changed files with 590 additions and 294 deletions
+49 -4
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
import threading
from .HTTP import HTTPClient from .HTTP import HTTPClient
@@ -16,6 +17,43 @@ class API:
def __init__(self, base_url: str, timeout: float = 10.0) -> None: def __init__(self, base_url: str, timeout: float = 10.0) -> None:
self.base_url = str(base_url or "").rstrip("/") self.base_url = str(base_url or "").rstrip("/")
self.timeout = float(timeout) self.timeout = float(timeout)
self._http_client: Optional[HTTPClient] = None
self._http_client_lock = threading.Lock()
def _get_http_client(self) -> HTTPClient:
"""Return a reusable opened HTTP client for this API instance."""
client = self._http_client
if client is not None and getattr(client, "_client", None) is not None:
return client
with self._http_client_lock:
client = self._http_client
if client is None:
client = HTTPClient(timeout=self.timeout)
self._http_client = client
if getattr(client, "_client", None) is None:
client.__enter__()
return client
def close(self) -> None:
client = self._http_client
if client is None:
return
with self._http_client_lock:
current = self._http_client
if current is None:
return
try:
current.__exit__(None, None, None)
except Exception:
pass
self._http_client = None
def __del__(self) -> None:
try:
self.close()
except Exception:
pass
def _get_json( def _get_json(
self, self,
@@ -25,8 +63,8 @@ class API:
) -> Dict[str, Any]: ) -> Dict[str, Any]:
url = f"{self.base_url}/{str(path or '').lstrip('/')}" url = f"{self.base_url}/{str(path or '').lstrip('/')}"
try: try:
with HTTPClient(timeout=self.timeout, headers=headers) as client: client = self._get_http_client()
response = client.get(url, params=params, allow_redirects=True) response = client.get(url, params=params, headers=headers, allow_redirects=True)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except Exception as exc: except Exception as exc:
@@ -41,8 +79,15 @@ class API:
) -> Dict[str, Any]: ) -> Dict[str, Any]:
url = f"{self.base_url}/{str(path or '').lstrip('/')}" url = f"{self.base_url}/{str(path or '').lstrip('/')}"
try: try:
with HTTPClient(timeout=self.timeout, headers=headers) as client: client = self._get_http_client()
response = client.post(url, json=json_data, params=params, allow_redirects=True) response = client.request(
"POST",
url,
json=json_data,
params=params,
headers=headers,
follow_redirects=True,
)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except Exception as exc: except Exception as exc:
+36
View File
@@ -27,6 +27,22 @@ from ProviderCore.base import Provider, SearchResult
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH") _EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
# Plugin instance cache keyed by (name, config_fingerprint)
_plugin_instance_cache: Dict[Tuple[str, str], Optional[Provider]] = {}
_plugin_cache_lock = __import__("threading").Lock()
def _config_fingerprint(config: Optional[Dict[str, Any]]) -> str:
"""Create a stable fingerprint of config for caching purposes."""
if config is None:
return "none"
try:
import json
normalized = json.dumps(config, sort_keys=True, default=str)
return hashlib.md5(normalized.encode()).hexdigest()[:16]
except Exception:
return "unknown"
def _class_supports_method( def _class_supports_method(
plugin_class: Type[Provider], plugin_class: Type[Provider],
@@ -603,14 +619,26 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
debug(f"[plugin] Unknown plugin: {name}") debug(f"[plugin] Unknown plugin: {name}")
return None return None
# Check cache first
cache_key = (str(name).strip().lower(), _config_fingerprint(config))
with _plugin_cache_lock:
if cache_key in _plugin_instance_cache:
return _plugin_instance_cache[cache_key]
try: try:
plugin = info.plugin_class(config) plugin = info.plugin_class(config)
if not plugin.validate(): if not plugin.validate():
debug(f"[plugin] Plugin '{name}' is not available") debug(f"[plugin] Plugin '{name}' is not available")
with _plugin_cache_lock:
_plugin_instance_cache[cache_key] = None
return None return None
with _plugin_cache_lock:
_plugin_instance_cache[cache_key] = plugin
return plugin return plugin
except Exception as exc: except Exception as exc:
debug(f"[plugin] Error initializing '{name}': {exc}") debug(f"[plugin] Error initializing '{name}': {exc}")
with _plugin_cache_lock:
_plugin_instance_cache[cache_key] = None
return None return None
@@ -860,6 +888,13 @@ def resolve_inline_filters(
return filters return filters
def clear_plugin_cache() -> None:
"""Clear the plugin instance cache. Useful for testing or config reloads."""
global _plugin_instance_cache
with _plugin_cache_lock:
_plugin_instance_cache.clear()
__all__ = [ __all__ = [
"PluginInfo", "PluginInfo",
"Provider", "Provider",
@@ -879,4 +914,5 @@ __all__ = [
"selection_auto_stage_for_table", "selection_auto_stage_for_table",
"plugin_inline_query_choices", "plugin_inline_query_choices",
"is_known_plugin_name", "is_known_plugin_name",
"clear_plugin_cache",
] ]
+38 -11
View File
@@ -10,18 +10,45 @@ import re
from typing import Any, Callable, Dict, List, Optional, Set, Tuple from typing import Any, Callable, Dict, List, Optional, Set, Tuple
from SYS.logger import debug from SYS.logger import debug
# Prompt-toolkit lexer types are optional at import time; fall back to lightweight # Prompt-toolkit lexer types are optional and expensive (~300ms). Use find_spec
# stubs if prompt_toolkit is not available so imports remain safe for testing. # to detect availability without importing, then lazy-load on first use.
try: import importlib.util as _importlib_util
from prompt_toolkit.document import Document _PTK_AVAILABLE: bool = _importlib_util.find_spec("prompt_toolkit") is not None
from prompt_toolkit.lexers import Lexer as _PTK_Lexer
except Exception: # pragma: no cover - optional dependency
Document = object # type: ignore
# Fallback to a simple object when prompt_toolkit is not available
_PTK_Lexer = object # type: ignore
# Expose a stable name used by the rest of the module _ptk_Document: Any = None
Lexer = _PTK_Lexer _ptk_Lexer: Any = None
def _get_ptk_Document() -> Any:
global _ptk_Document
if _ptk_Document is None:
if _PTK_AVAILABLE:
from prompt_toolkit.document import Document as _Doc
_ptk_Document = _Doc
else:
_ptk_Document = object
return _ptk_Document
def _get_ptk_Lexer() -> Any:
global _ptk_Lexer
if _ptk_Lexer is None:
if _PTK_AVAILABLE:
from prompt_toolkit.lexers import Lexer as _Lex
_ptk_Lexer = _Lex
else:
_ptk_Lexer = object
return _ptk_Lexer
# Stable aliases: these resolve lazily the first time they are accessed.
# Code that does `isinstance(x, Document)` or `class Foo(Lexer)` at class-body
# time needs the real object, so we keep module-level names that proxy to the
# lazy getters via __getattr__ on the module. Callers that reference
# Document/Lexer INSIDE functions will always get the real class.
# Populate the module-level names now so that class bodies below can inherit.
Document: Any = _get_ptk_Document()
Lexer: Any = _get_ptk_Lexer()
# Pre-compiled regexes for the lexer (avoid recompiling on every call) # Pre-compiled regexes for the lexer (avoid recompiling on every call)
TOKEN_PATTERN = re.compile( TOKEN_PATTERN = re.compile(
+22 -11
View File
@@ -1,13 +1,21 @@
"""Unified logging utility for automatic file and function name tracking.""" """Unified logging utility for automatic file and function name tracking."""
import sys import sys
import inspect
import logging import logging
import threading import threading
from pathlib import Path from pathlib import Path
from typing import Any, Optional, Sequence from typing import Any, Optional, Sequence
from SYS.rich_display import console_for # SYS.rich_display deferred: rich (~100ms) loaded lazily on first log output.
_rich_display_mod: Any = None
def _console_for(file):
global _rich_display_mod
if _rich_display_mod is None:
import SYS.rich_display as _m
_rich_display_mod = _m
return _rich_display_mod.console_for(file)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -73,7 +81,8 @@ def _is_rich_renderable(value: Any) -> bool:
def _caller_location(depth: int = 1) -> tuple[str, str]: def _caller_location(depth: int = 1) -> tuple[str, str]:
frame = inspect.currentframe() import inspect as _inspect
frame = _inspect.currentframe()
current = frame current = frame
try: try:
for _ in range(max(0, int(depth))): for _ in range(max(0, int(depth))):
@@ -160,7 +169,7 @@ def debug(*args, **kwargs) -> None:
if len(args) == 1 and _is_rich_renderable(args[0]): if len(args) == 1 and _is_rich_renderable(args[0]):
renderable = args[0] renderable = args[0]
console_for(target_file).print(renderable) _console_for(target_file).print(renderable)
file_name, func_name = _caller_location(depth=1) file_name, func_name = _caller_location(depth=1)
caller_name = f"{file_name}.{func_name}" if file_name and func_name else "" caller_name = f"{file_name}.{func_name}" if file_name and func_name else ""
_debug_db_log(caller_name=caller_name, message=f"<rich:{type(renderable).__name__}>") _debug_db_log(caller_name=caller_name, message=f"<rich:{type(renderable).__name__}>")
@@ -200,7 +209,8 @@ def debug_inspect(
# Compute caller prefix (same as log()). # Compute caller prefix (same as log()).
prefix = None prefix = None
frame = inspect.currentframe() import inspect as _inspect
frame = _inspect.currentframe()
if frame is not None and frame.f_back is not None: if frame is not None and frame.f_back is not None:
caller_frame = frame.f_back caller_frame = frame.f_back
try: try:
@@ -215,7 +225,7 @@ def debug_inspect(
# Render. # Render.
from rich import inspect as rich_inspect from rich import inspect as rich_inspect
console = console_for(file) console = _console_for(file)
# If the caller provides a title, treat it as authoritative. # If the caller provides a title, treat it as authoritative.
# Only fall back to the automatic [file.func] prefix when no title is supplied. # Only fall back to the automatic [file.func] prefix when no title is supplied.
effective_title = title effective_title = title
@@ -266,12 +276,13 @@ def log(*args, **kwargs) -> None:
add_prefix = _DEBUG_ENABLED add_prefix = _DEBUG_ENABLED
# Get the calling frame # Get the calling frame
frame = inspect.currentframe() import inspect as _inspect
frame = _inspect.currentframe()
if frame is None: if frame is None:
file = kwargs.pop("file", sys.stdout) file = kwargs.pop("file", sys.stdout)
sep = kwargs.pop("sep", " ") sep = kwargs.pop("sep", " ")
end = kwargs.pop("end", "\n") end = kwargs.pop("end", "\n")
console_for(file).print(*args, sep=sep, end=end) _console_for(file).print(*args, sep=sep, end=end)
return return
caller_frame = frame.f_back caller_frame = frame.f_back
@@ -279,7 +290,7 @@ def log(*args, **kwargs) -> None:
file = kwargs.pop("file", sys.stdout) file = kwargs.pop("file", sys.stdout)
sep = kwargs.pop("sep", " ") sep = kwargs.pop("sep", " ")
end = kwargs.pop("end", "\n") end = kwargs.pop("end", "\n")
console_for(file).print(*args, sep=sep, end=end) _console_for(file).print(*args, sep=sep, end=end)
return return
try: try:
@@ -302,9 +313,9 @@ def log(*args, **kwargs) -> None:
end = kwargs.pop("end", "\n") end = kwargs.pop("end", "\n")
if add_prefix: if add_prefix:
prefix = f"[{file_name}.{func_name}]" prefix = f"[{file_name}.{func_name}]"
console_for(file).print(prefix, *args, sep=sep, end=end) _console_for(file).print(prefix, *args, sep=sep, end=end)
else: else:
console_for(file).print(*args, sep=sep, end=end) _console_for(file).print(*args, sep=sep, end=end)
# Log to database if available # Log to database if available
if _DB_LOGGER: if _DB_LOGGER:
+80 -56
View File
@@ -1,5 +1,7 @@
"""Data models for the pipeline.""" """Data models for the pipeline."""
from __future__ import annotations
import datetime import datetime
import hashlib import hashlib
import inspect import inspect
@@ -16,23 +18,9 @@ from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Protocol, TextIO from typing import Any, Callable, Dict, List, Optional, Protocol, TextIO
from rich.console import Console # rich imports are deferred to avoid ~100ms startup cost.
from rich.console import ConsoleOptions # Classes in this module that use rich types (ProgressBar, PipelineLiveProgress)
from rich.console import Group # import them lazily inside their method bodies at first use.
from rich.live import Live
from rich.panel import Panel
from rich.progress import (
BarColumn,
DownloadColumn,
Progress,
SpinnerColumn,
TaskID,
TaskProgressColumn,
TextColumn,
TimeRemainingColumn,
TimeElapsedColumn,
TransferSpeedColumn,
)
@dataclass(slots=True) @dataclass(slots=True)
@@ -440,6 +428,42 @@ def _sanitise_for_json(
return repr(value) return repr(value)
def _import_rich() -> Any:
"""Lazy-load rich types used by ProgressBar and PipelineLiveProgress."""
import rich.console as _rc
import rich.live as _rl
import rich.panel as _rp
import rich.progress as _rprog
# Return a namespace-like object with the types we need
class _Rich:
Console = _rc.Console
ConsoleOptions = _rc.ConsoleOptions
Group = _rc.Group
Live = _rl.Live
Panel = _rp.Panel
Progress = _rprog.Progress
BarColumn = _rprog.BarColumn
DownloadColumn = _rprog.DownloadColumn
SpinnerColumn = _rprog.SpinnerColumn
TaskID = _rprog.TaskID
TaskProgressColumn = _rprog.TaskProgressColumn
TextColumn = _rprog.TextColumn
TimeRemainingColumn = _rprog.TimeRemainingColumn
TimeElapsedColumn = _rprog.TimeElapsedColumn
TransferSpeedColumn = _rprog.TransferSpeedColumn
return _Rich
_rich: Any = None # cached after first call
def _r() -> Any:
global _rich
if _rich is None:
_rich = _import_rich()
return _rich
class ProgressBar: class ProgressBar:
"""Rich progress helper for byte-based transfers. """Rich progress helper for byte-based transfers.
@@ -521,16 +545,16 @@ class ProgressBar:
console = stderr_console() console = stderr_console()
except Exception: except Exception:
logger.exception("Failed to acquire shared stderr Console from SYS.rich_display; using fallback Console") logger.exception("Failed to acquire shared stderr Console from SYS.rich_display; using fallback Console")
console = Console(file=stream) console = _r().Console(file=stream)
else: else:
console = Console(file=stream) console = _r().Console(file=stream)
progress = Progress( progress = _r().Progress(
TextColumn("[progress.description]{task.description}"), _r().TextColumn("[progress.description]{task.description}"),
BarColumn(), _r().BarColumn(),
TaskProgressColumn(), _r().TaskProgressColumn(),
DownloadColumn(), _r().DownloadColumn(),
TransferSpeedColumn(), _r().TransferSpeedColumn(),
TimeRemainingColumn(), _r().TimeRemainingColumn(),
console=console, console=console,
transient=True, transient=True,
) )
@@ -867,7 +891,7 @@ class PipelineLiveProgress:
overall = self._overall overall = self._overall
if pipe_progress is None or transfers is None or overall is None: if pipe_progress is None or transfers is None or overall is None:
# Not started (or stopped). # Not started (or stopped).
yield Panel("", title="Pipeline", expand=False) yield _r().Panel("", title="Pipeline", expand=False)
return return
body_parts: List[Any] = [pipe_progress] body_parts: List[Any] = [pipe_progress]
@@ -875,8 +899,8 @@ class PipelineLiveProgress:
body_parts.append(status) body_parts.append(status)
body_parts.append(transfers) body_parts.append(transfers)
yield Group( yield _r().Group(
Panel(Group(*body_parts), _r().Panel(_r().Group(*body_parts),
title=self._title_text(), title=self._title_text(),
expand=False), expand=False),
overall overall
@@ -895,8 +919,8 @@ class PipelineLiveProgress:
if status is not None and self._status_tasks: if status is not None and self._status_tasks:
body_parts.append(status) body_parts.append(status)
body_parts.append(transfers) body_parts.append(transfers)
return Group( return _r().Group(
Panel(Group(*body_parts), _r().Panel(_r().Group(*body_parts),
title=self._title_text(), title=self._title_text(),
expand=False), expand=False),
overall overall
@@ -911,55 +935,55 @@ class PipelineLiveProgress:
# IMPORTANT: use the shared stderr Console instance so that any # IMPORTANT: use the shared stderr Console instance so that any
# `stderr_console().print(...)` calls from inside cmdlets (e.g. preflight # `stderr_console().print(...)` calls from inside cmdlets (e.g. preflight
# tables/prompts in download-file) cooperate with Rich Live rendering. # tables/prompts in download-file) cooperate with Rich Live rendering.
# If we create a separate Console(file=sys.stderr), output will fight for # If we create a separate _r().Console(file=sys.stderr), output will fight for
# terminal cursor control and appear "blocked"/truncated. # terminal cursor control and appear "blocked"/truncated.
from SYS.rich_display import stderr_console from SYS.rich_display import stderr_console
self._console = stderr_console() self._console = stderr_console()
# Persistent per-pipe bars. # Persistent per-pipe bars.
self._pipe_progress = Progress( self._pipe_progress = _r().Progress(
TextColumn("{task.description}"), _r().TextColumn("{task.description}"),
TimeElapsedColumn(), _r().TimeElapsedColumn(),
BarColumn(), _r().BarColumn(),
TaskProgressColumn(), _r().TaskProgressColumn(),
console=self._console, console=self._console,
transient=False, transient=False,
) )
# Transient, per-item spinner for the currently-active subtask. # Transient, per-item spinner for the currently-active subtask.
self._subtasks = Progress( self._subtasks = _r().Progress(
TextColumn(" "), _r().TextColumn(" "),
SpinnerColumn("simpleDots"), _r().SpinnerColumn("simpleDots"),
TextColumn("{task.description}"), _r().TextColumn("{task.description}"),
console=self._console, console=self._console,
transient=False, transient=False,
) )
# Status line below the pipe bars. Kept simple (no extra bar) so it # Status line below the pipe bars. Kept simple (no extra bar) so it
# doesn't visually offset the main pipe bar columns. # doesn't visually offset the main pipe bar columns.
self._status = Progress( self._status = _r().Progress(
TextColumn(" [bold]└─ {task.description}[/bold]"), _r().TextColumn(" [bold]└─ {task.description}[/bold]"),
console=self._console, console=self._console,
transient=False, transient=False,
) )
# Byte-based transfer bars (download/upload) integrated into the Live view. # Byte-based transfer bars (download/upload) integrated into the Live view.
self._transfers = Progress( self._transfers = _r().Progress(
TextColumn(" {task.description}"), _r().TextColumn(" {task.description}"),
BarColumn(), _r().BarColumn(),
TaskProgressColumn(), _r().TaskProgressColumn(),
DownloadColumn(), _r().DownloadColumn(),
TransferSpeedColumn(), _r().TransferSpeedColumn(),
TimeRemainingColumn(), _r().TimeRemainingColumn(),
console=self._console, console=self._console,
transient=False, transient=False,
) )
self._overall = Progress( self._overall = _r().Progress(
TimeElapsedColumn(), _r().TimeElapsedColumn(),
BarColumn(), _r().BarColumn(),
TextColumn("{task.description}"), _r().TextColumn("{task.description}"),
console=self._console, console=self._console,
transient=False, transient=False,
) )
@@ -982,7 +1006,7 @@ class PipelineLiveProgress:
len(self._pipe_labels)), len(self._pipe_labels)),
) )
self._live = Live( self._live = _r().Live(
self, self,
console=self._console, console=self._console,
refresh_per_second=10, refresh_per_second=10,
@@ -1011,7 +1035,7 @@ class PipelineLiveProgress:
# Not initialized yet; start fresh. # Not initialized yet; start fresh.
self.start() self.start()
return return
self._live = Live( self._live = _r().Live(
self, self,
console=self._console, console=self._console,
refresh_per_second=10, refresh_per_second=10,
+49 -35
View File
@@ -13,11 +13,41 @@ from SYS.models import PipelineStageContext
from SYS.logger import log, debug, debug_panel, is_debug_enabled from SYS.logger import log, debug, debug_panel, is_debug_enabled
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from SYS.worker import WorkerManagerRegistry, WorkerStages # SYS.worker deferred: ffmpeg+attr+rich (~260ms) loaded lazily on first pipeline run.
from SYS.cli_parsing import SelectionSyntax, SelectionFilterSyntax _worker_mod: Any = None
from SYS.rich_display import stdout_console # SYS.cli_parsing deferred: prompt_toolkit (~300ms) loaded lazily on first selection.
from SYS.background_notifier import ensure_background_notifier _cli_parsing_mod: Any = None
from SYS.result_table import Table
def _worker():
global _worker_mod
if _worker_mod is None:
import SYS.worker as _m
_worker_mod = _m
return _worker_mod
def _cli_parsing():
global _cli_parsing_mod
if _cli_parsing_mod is None:
import SYS.cli_parsing as _m
_cli_parsing_mod = _m
return _cli_parsing_mod
# SYS.rich_display deferred: rich (~100ms) loaded lazily on first console output.
# SYS.background_notifier deferred: rich/attr/ffmpeg loaded lazily on first notifier use.
# SYS.result_table deferred: textual (~140ms) loaded lazily on first Table use.
_result_table_mod: Any = None
def _result_table():
global _result_table_mod
if _result_table_mod is None:
from SYS.result_table import Table as _T
_result_table_mod = _T
return _result_table_mod
import re import re
from datetime import datetime from datetime import datetime
from SYS.cmdlet_catalog import import_cmd_module from SYS.cmdlet_catalog import import_cmd_module
@@ -680,12 +710,14 @@ def set_last_result_table(
""" """
state = _get_pipeline_state() state = _get_pipeline_state()
# Push current table to history before replacing # Push current table to history before replacing.
# No .copy() needed: last_result_items is about to be replaced by reference,
# not mutated in place, so the old list reference is safe to keep in history.
if state.last_result_table is not None: if state.last_result_table is not None:
state.result_table_history.append( state.result_table_history.append(
( (
state.last_result_table, state.last_result_table,
state.last_result_items.copy(), state.last_result_items,
state.last_result_subject, state.last_result_subject,
) )
) )
@@ -724,26 +756,6 @@ def set_last_result_table(
logger.exception("Failed to sort result_table and reorder items") logger.exception("Failed to sort result_table and reorder items")
if (
result_table is not None
and hasattr(result_table, "sort_by_title")
and not getattr(result_table, "preserve_order", False)
):
try:
result_table.sort_by_title()
# Re-order items list to match the sorted table
if state.display_items and hasattr(result_table, "rows"):
sorted_items: List[Any] = []
for row in result_table.rows:
src_idx = getattr(row, "source_index", None)
if isinstance(src_idx, int) and 0 <= src_idx < len(state.display_items):
sorted_items.append(state.display_items[src_idx])
if len(sorted_items) == len(result_table.rows):
state.display_items = sorted_items
except Exception:
logger.exception("Failed to sort overlay result_table and reorder items")
def set_last_result_table_overlay( def set_last_result_table_overlay(
result_table: Optional[Any], result_table: Optional[Any],
items: Optional[List[Any]] = None, items: Optional[List[Any]] = None,
@@ -1423,7 +1435,7 @@ class PipelineExecutor:
new_first_stage: List[str] = [] new_first_stage: List[str] = []
for token in first_stage_tokens: for token in first_stage_tokens:
if token.startswith("@"): # selection if token.startswith("@"): # selection
selection = SelectionSyntax.parse(token) selection = _cli_parsing().SelectionSyntax.parse(token)
if selection is not None: if selection is not None:
first_stage_selection_indices = sorted( first_stage_selection_indices = sorted(
[i - 1 for i in selection] [i - 1 for i in selection]
@@ -1848,6 +1860,7 @@ class PipelineExecutor:
} }
if output_fn: if output_fn:
kwargs["output"] = output_fn kwargs["output"] = output_fn
from SYS.background_notifier import ensure_background_notifier
ensure_background_notifier(worker_manager, **kwargs) ensure_background_notifier(worker_manager, **kwargs)
except Exception: except Exception:
logger.exception("Failed to enable background notifier for session_worker_ids=%r", session_worker_ids) logger.exception("Failed to enable background notifier for session_worker_ids=%r", session_worker_ids)
@@ -2633,9 +2646,9 @@ class PipelineExecutor:
) )
piped_result: Any = None piped_result: Any = None
worker_manager = WorkerManagerRegistry.ensure(config) worker_manager = _worker().WorkerManagerRegistry.ensure(config)
pipeline_text = " | ".join(" ".join(stage) for stage in stages) pipeline_text = " | ".join(" ".join(stage) for stage in stages)
pipeline_session = WorkerStages.begin_pipeline( pipeline_session = _worker().WorkerStages.begin_pipeline(
worker_manager, worker_manager,
pipeline_text=pipeline_text, pipeline_text=pipeline_text,
config=config config=config
@@ -2790,8 +2803,8 @@ class PipelineExecutor:
if cmd_name.startswith("@"): # selection stage if cmd_name.startswith("@"): # selection stage
selection_token = raw_stage_name selection_token = raw_stage_name
selection = SelectionSyntax.parse(selection_token) selection = _cli_parsing().SelectionSyntax.parse(selection_token)
filter_spec = SelectionFilterSyntax.parse(selection_token) filter_spec = _cli_parsing().SelectionFilterSyntax.parse(selection_token)
is_select_all = selection_token.strip() == "@*" is_select_all = selection_token.strip() == "@*"
if selection is None and filter_spec is None and not is_select_all: if selection is None and filter_spec is None and not is_select_all:
print(f"Invalid selection: {selection_token}\n") print(f"Invalid selection: {selection_token}\n")
@@ -2849,7 +2862,7 @@ class PipelineExecutor:
elif filter_spec is not None: elif filter_spec is not None:
selected_indices = [ selected_indices = [
i for i, item in enumerate(items_list) i for i, item in enumerate(items_list)
if SelectionFilterSyntax.matches(item, filter_spec) if _cli_parsing().SelectionFilterSyntax.matches(item, filter_spec)
] ]
else: else:
selected_indices = sorted( selected_indices = sorted(
@@ -2894,7 +2907,7 @@ class PipelineExecutor:
if base_table is not None and hasattr(base_table, "copy_with_title"): if base_table is not None and hasattr(base_table, "copy_with_title"):
new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results")
else: else:
new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results") new_table = _result_table()(getattr(base_table, "title", "") if base_table is not None else "Results")
try: try:
if base_table is not None and getattr(base_table, "table", None): if base_table is not None and getattr(base_table, "table", None):
@@ -2918,6 +2931,7 @@ class PipelineExecutor:
logger.exception("Failed to set last_result_table_overlay for filter selection") logger.exception("Failed to set last_result_table_overlay for filter selection")
try: try:
from SYS.rich_display import stdout_console
stdout_console().print() stdout_console().print()
stdout_console().print(new_table) stdout_console().print(new_table)
except Exception: except Exception:
@@ -3118,7 +3132,7 @@ class PipelineExecutor:
pipe_idx = pipe_index_by_stage.get(stage_index) pipe_idx = pipe_index_by_stage.get(stage_index)
overlay_table: Any | None = None overlay_table: Any | None = None
session = WorkerStages.begin_stage( session = _worker().WorkerStages.begin_stage(
worker_manager, worker_manager,
cmd_name=cmd_name, cmd_name=cmd_name,
stage_tokens=stage_tokens, stage_tokens=stage_tokens,
+71 -44
View File
@@ -18,20 +18,47 @@ from pathlib import Path
import json import json
import re import re
from rich.box import SIMPLE # rich imports are deferred to avoid ~100ms startup cost.
from rich.console import Group # All rich types are only needed inside method bodies, so we lazily import on first use.
from rich.panel import Panel _rich_mod: Any = None
from rich.prompt import Prompt
from rich.table import Table as RichTable
from rich.text import Text
# Optional Textual imports - graceful fallback if not available
try:
from textual.widgets import Tree
TEXTUAL_AVAILABLE = True def _rich():
except ImportError: global _rich_mod
TEXTUAL_AVAILABLE = False if _rich_mod is None:
import types as _types
_m = _types.SimpleNamespace()
from rich.box import SIMPLE as _SIMPLE
from rich.console import Group as _Group
from rich.panel import Panel as _Panel
from rich.prompt import Prompt as _Prompt
from rich.table import Table as _RichTable
from rich.text import Text as _Text
_m.SIMPLE = _SIMPLE
_m.Group = _Group
_m.Panel = _Panel
_m.Prompt = _Prompt
_m.RichTable = _RichTable
_m.Text = _Text
_rich_mod = _m
return _rich_mod
# Optional Textual imports - lazily loaded to avoid pulling in ~300ms of textual
# at import time when the TUI is not being used.
import importlib.util as _importlib_util
TEXTUAL_AVAILABLE: bool = _importlib_util.find_spec("textual") is not None
# Tree is populated lazily on first call to build_metadata_tree().
_textual_Tree: Any = None
def _get_textual_Tree() -> Any:
global _textual_Tree
if _textual_Tree is None:
from textual.widgets import Tree as _Tree
_textual_Tree = _Tree
return _textual_Tree
# Import ResultModel from the API for typing; avoid runtime redefinition issues # Import ResultModel from the API for typing; avoid runtime redefinition issues
@@ -1591,11 +1618,11 @@ class Table:
panel_style = get_result_table_panel_style({"table_appearance": appearance_mode}) panel_style = get_result_table_panel_style({"table_appearance": appearance_mode})
if not self.rows: if not self.rows:
empty = Text("No results") empty = _rich().Text("No results")
return ( return (
Panel( _rich().Panel(
empty, empty,
title=Text(str(self.title), style=header_style), title=_rich().Text(str(self.title), style=header_style),
border_style=border_style, border_style=border_style,
padding=(0, 0), padding=(0, 0),
expand=False, expand=False,
@@ -1613,7 +1640,7 @@ class Table:
seen.add(col.name) seen.add(col.name)
col_names.append(col.name) col_names.append(col.name)
table = RichTable( table = _rich().RichTable(
show_header=True, show_header=True,
header_style=header_style, header_style=header_style,
border_style=border_style, border_style=border_style,
@@ -1661,12 +1688,12 @@ class Table:
) )
if self.title or self.header_lines: if self.title or self.header_lines:
header_bits = [Text(line) for line in (self.header_lines or [])] header_bits = [_rich().Text(line) for line in (self.header_lines or [])]
renderable = Group(*header_bits, table) if header_bits else table renderable = _rich().Group(*header_bits, table) if header_bits else table
return ( return (
Panel( _rich().Panel(
renderable, renderable,
title=Text(str(self.title), style=header_style), title=_rich().Text(str(self.title), style=header_style),
border_style=border_style, border_style=border_style,
padding=(0, 0), padding=(0, 0),
expand=False, expand=False,
@@ -1777,7 +1804,7 @@ class Table:
from SYS.rich_display import stdout_console from SYS.rich_display import stdout_console
stdout_console().print(self) stdout_console().print(self)
stdout_console().print(Panel(Text("Selection is disabled for this table."))) stdout_console().print(_rich().Panel(_rich().Text("Selection is disabled for this table.")))
return None return None
# Display the table # Display the table
@@ -1789,11 +1816,11 @@ class Table:
while True: while True:
try: try:
if accept_args: if accept_args:
choice = Prompt.ask( choice = _rich().Prompt.ask(
f"{prompt} (e.g., '5' or '2 -storage hydrus' or 'q' to quit)" f"{prompt} (e.g., '5' or '2 -storage hydrus' or 'q' to quit)"
).strip() ).strip()
else: else:
choice = Prompt.ask( choice = _rich().Prompt.ask(
f"{prompt} (e.g., '5' or '3-5' or '1,3,5' or 'q' to quit)" f"{prompt} (e.g., '5' or '3-5' or '1,3,5' or 'q' to quit)"
).strip() ).strip()
@@ -1806,8 +1833,8 @@ class Table:
if result is not None: if result is not None:
return result return result
stdout_console().print( stdout_console().print(
Panel( _rich().Panel(
Text( _rich().Text(
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')." "Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
) )
) )
@@ -1818,8 +1845,8 @@ class Table:
if selected_indices is not None: if selected_indices is not None:
return selected_indices return selected_indices
stdout_console().print( stdout_console().print(
Panel( _rich().Panel(
Text( _rich().Text(
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit." "Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
) )
) )
@@ -1827,16 +1854,16 @@ class Table:
except (ValueError, EOFError): except (ValueError, EOFError):
if accept_args: if accept_args:
stdout_console().print( stdout_console().print(
Panel( _rich().Panel(
Text( _rich().Text(
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')." "Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
) )
) )
) )
else: else:
stdout_console().print( stdout_console().print(
Panel( _rich().Panel(
Text( _rich().Text(
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit." "Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
) )
) )
@@ -2468,12 +2495,12 @@ class ItemDetailView(Table):
from rich.text import Text from rich.text import Text
# 1. Create Detail Grid (matching rich_display.py style) # 1. Create Detail Grid (matching rich_display.py style)
details_table = RichTable.grid(expand=True, padding=(0, 2)) details_table = _rich().RichTable.grid(expand=True, padding=(0, 2))
details_table.add_column("Key", style="cyan", justify="right", width=15) details_table.add_column("Key", style="cyan", justify="right", width=15)
details_table.add_column("Value", style="white") details_table.add_column("Value", style="white")
def _render_tag_text(tag_value: Any) -> Text: def _render_tag_text(tag_value: Any) -> Text:
tag_text = Text() tag_text = _rich().Text()
tag_text.append("#", style="dim") tag_text.append("#", style="dim")
raw = str(tag_value or "") raw = str(tag_value or "")
@@ -2497,17 +2524,17 @@ class ItemDetailView(Table):
renderables.append(_render_tag_text(tag)) renderables.append(_render_tag_text(tag))
if freeform_tags: if freeform_tags:
freeform_grid = RichTable.grid(expand=True, padding=(0, 2)) freeform_grid = _rich().RichTable.grid(expand=True, padding=(0, 2))
for _ in range(3): for _ in range(3):
freeform_grid.add_column(ratio=1) freeform_grid.add_column(ratio=1)
for row_values in _chunk_detail_tags(freeform_tags, 3): for row_values in _chunk_detail_tags(freeform_tags, 3):
cells = [_render_tag_text(tag) for tag in row_values] cells = [_render_tag_text(tag) for tag in row_values]
while len(cells) < 3: while len(cells) < 3:
cells.append(Text("")) cells.append(_rich().Text(""))
freeform_grid.add_row(*cells) freeform_grid.add_row(*cells)
renderables.append(freeform_grid) renderables.append(freeform_grid)
return Group(*renderables) return _rich().Group(*renderables)
def _has_renderable_value(value: Any) -> bool: def _has_renderable_value(value: Any) -> bool:
if value is None: if value is None:
@@ -2596,9 +2623,9 @@ class ItemDetailView(Table):
header_style = get_result_table_header_style() header_style = get_result_table_header_style()
border_style = get_result_table_border_style() border_style = get_result_table_border_style()
detail_title = str(self.detail_title or "Item Details").strip() or "Item Details" detail_title = str(self.detail_title or "Item Details").strip() or "Item Details"
elements.append(Panel( elements.append(_rich().Panel(
details_table, details_table,
title=Text(detail_title, style=header_style), title=_rich().Text(detail_title, style=header_style),
border_style=border_style, border_style=border_style,
padding=(1, 2) padding=(1, 2)
)) ))
@@ -2606,10 +2633,10 @@ class ItemDetailView(Table):
if results_renderable: if results_renderable:
# If it's a Panel already (from super().to_rich() with title), use it directly # If it's a Panel already (from super().to_rich() with title), use it directly
# but force the border style to the result-table standard for consistency # but force the border style to the result-table standard for consistency
if isinstance(results_renderable, Panel): if isinstance(results_renderable, _rich().Panel):
results_renderable.border_style = get_result_table_border_style() results_renderable.border_style = get_result_table_border_style()
if results_renderable.title: if results_renderable.title:
results_renderable.title = Text( results_renderable.title = _rich().Text(
str(results_renderable.title), str(results_renderable.title),
style=get_result_table_header_style(), style=get_result_table_header_style(),
) )
@@ -2622,13 +2649,13 @@ class ItemDetailView(Table):
display_title = original_title display_title = original_title
# Add a bit of padding # Add a bit of padding
results_group = Group(Text(""), results_renderable, Text("")) results_group = _rich().Group(_rich().Text(""), results_renderable, _rich().Text(""))
elements.append( elements.append(
Panel( _rich().Panel(
results_group, results_group,
title=Text(str(display_title), style=get_result_table_header_style()), title=_rich().Text(str(display_title), style=get_result_table_header_style()),
border_style=get_result_table_border_style(), border_style=get_result_table_border_style(),
) )
) )
return Group(*elements) return _rich().Group(*elements)
+26 -17
View File
@@ -13,42 +13,49 @@ import contextlib
import sys import sys
from typing import Any, Iterator, TextIO, List, Dict, Optional, Tuple, cast from typing import Any, Iterator, TextIO, List, Dict, Optional, Tuple, cast
from rich.console import Console
from rich.panel import Panel
from rich.text import Text
from pathlib import Path from pathlib import Path
from SYS.utils import expand_path from SYS.utils import expand_path
# Configure Rich pretty-printing to avoid truncating long strings (hashes/paths). # rich imports are deferred to first Console use to avoid ~100ms startup cost.
# This is version-safe: older Rich versions may not support the max_* arguments. # They are loaded the first time any Console function is called.
try:
from rich.pretty import install as _pretty_install
_STDOUT_CONSOLE: Any = None
_STDERR_CONSOLE: Any = None
def _ensure_consoles() -> None:
global _STDOUT_CONSOLE, _STDERR_CONSOLE
if _STDOUT_CONSOLE is None:
from rich.console import Console
from rich.pretty import install as _pretty_install
try: try:
_pretty_install(max_string=100_000, max_length=100_000) _pretty_install(max_string=100_000, max_length=100_000)
except TypeError: except TypeError:
_pretty_install() _pretty_install()
except Exception: except Exception:
from SYS.logger import logger pass
logger.exception("Failed to configure rich pretty-printing") _STDOUT_CONSOLE = Console(file=sys.stdout)
_STDERR_CONSOLE = Console(file=sys.stderr)
_STDOUT_CONSOLE = Console(file=sys.stdout)
_STDERR_CONSOLE = Console(file=sys.stderr)
def stdout_console() -> Console: def stdout_console() -> Any:
_ensure_consoles()
return _STDOUT_CONSOLE return _STDOUT_CONSOLE
def stderr_console() -> Console: def stderr_console() -> Any:
_ensure_consoles()
return _STDERR_CONSOLE return _STDERR_CONSOLE
def console_for(file: TextIO | None) -> Console: def console_for(file: Any) -> Any:
if file is None or file is sys.stdout: if file is None or file is sys.stdout:
_ensure_consoles()
return _STDOUT_CONSOLE return _STDOUT_CONSOLE
if file is sys.stderr: if file is sys.stderr:
_ensure_consoles()
return _STDERR_CONSOLE return _STDERR_CONSOLE
from rich.console import Console
return Console(file=file) return Console(file=file)
@@ -57,7 +64,7 @@ def rprint(renderable: Any = "", *, file: TextIO | None = None) -> None:
@contextlib.contextmanager @contextlib.contextmanager
def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]: def capture_rich_output(*, stdout: Any, stderr: Any) -> Iterator[None]:
"""Temporarily redirect Rich output helpers to provided streams. """Temporarily redirect Rich output helpers to provided streams.
Note: `stdout_console()` / `stderr_console()` use global Console instances, Note: `stdout_console()` / `stderr_console()` use global Console instances,
@@ -65,9 +72,11 @@ def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]:
""" """
global _STDOUT_CONSOLE, _STDERR_CONSOLE global _STDOUT_CONSOLE, _STDERR_CONSOLE
_ensure_consoles()
previous_stdout = _STDOUT_CONSOLE previous_stdout = _STDOUT_CONSOLE
previous_stderr = _STDERR_CONSOLE previous_stderr = _STDERR_CONSOLE
from rich.console import Console
try: try:
_STDOUT_CONSOLE = Console(file=stdout) _STDOUT_CONSOLE = Console(file=stdout)
_STDERR_CONSOLE = Console(file=stderr) _STDERR_CONSOLE = Console(file=stderr)
+19 -7
View File
@@ -4,13 +4,7 @@ from __future__ import annotations
import json import json
import hashlib import hashlib
import subprocess
import shutil import shutil
try:
import ffmpeg # type: ignore
except Exception:
ffmpeg = None # type: ignore
import os import os
import base64 import base64
import logging import logging
@@ -23,6 +17,22 @@ from urllib.parse import urlparse
from SYS.utils_constant import mime_maps from SYS.utils_constant import mime_maps
_ffmpeg_mod: Any = None
_ffmpeg_checked = False
def _get_ffmpeg():
"""Lazily return the ffmpeg module, or None if unavailable."""
global _ffmpeg_mod, _ffmpeg_checked
if not _ffmpeg_checked:
try:
import ffmpeg as _f # type: ignore
_ffmpeg_mod = _f
except Exception:
_ffmpeg_mod = None
_ffmpeg_checked = True
return _ffmpeg_mod
try: try:
import cbor2 import cbor2
except ImportError: except ImportError:
@@ -191,6 +201,7 @@ def ffprobe(file_path: str) -> dict:
probe = None probe = None
# Try python ffmpeg module first # Try python ffmpeg module first
ffmpeg = _get_ffmpeg()
if ffmpeg is not None: if ffmpeg is not None:
try: try:
probe = ffmpeg.probe(file_path) probe = ffmpeg.probe(file_path)
@@ -203,7 +214,8 @@ def ffprobe(file_path: str) -> dict:
ffprobe_cmd = shutil.which("ffprobe") ffprobe_cmd = shutil.which("ffprobe")
if ffprobe_cmd: if ffprobe_cmd:
try: try:
proc = subprocess.run( import subprocess as _subprocess
proc = _subprocess.run(
[ [
ffprobe_cmd, ffprobe_cmd,
"-v", "-v",
+82 -22
View File
@@ -40,11 +40,58 @@ build_pipeline_preview = sh.build_pipeline_preview
get_field = sh.get_field get_field = sh.get_field
from SYS.utils import sha256_file, unique_path, sanitize_filename from SYS.utils import sha256_file, unique_path, sanitize_filename
from SYS.metadata import write_metadata
# Canonical supported filetypes for all stores/cmdlets # Canonical supported filetypes for all stores/cmdlets
SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS
class _CommandDependencies:
"""Command-scope cache for Store and plugin instances to avoid repeated instantiation."""
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
self._store: Optional[Store] = None
self._plugins: Dict[str, Any] = {}
def get_store(self) -> Optional[Store]:
"""Lazily initialize and return the command-scope Store instance."""
if self._store is None:
try:
self._store = Store(self.config)
except Exception:
self._store = None
return self._store
def get_plugin(self, name: str) -> Optional[Any]:
"""Cached plugin lookup by name."""
from ProviderCore.registry import get_plugin
norm_name = str(name or "").strip().lower()
if not norm_name:
return None
if norm_name in self._plugins:
return self._plugins[norm_name]
plugin = get_plugin(norm_name, self.config)
self._plugins[norm_name] = plugin
return plugin
def get_plugin_with_capability(self, name: str, capability: str) -> Optional[Any]:
"""Cached plugin lookup with capability check."""
from ProviderCore.registry import get_plugin_with_capability
norm_name = str(name or "").strip().lower()
if not norm_name:
return None
cache_key = f"{norm_name}#{capability}"
if cache_key in self._plugins:
return self._plugins[cache_key]
plugin = get_plugin_with_capability(norm_name, capability, self.config)
self._plugins[cache_key] = plugin
return plugin
DEBUG_PIPE_NOTE_PREVIEW_LENGTH = 256 DEBUG_PIPE_NOTE_PREVIEW_LENGTH = 256
# Protocol schemes that identify a remote resource / not a local file path. # Protocol schemes that identify a remote resource / not a local file path.
@@ -220,11 +267,9 @@ class Add_File(Cmdlet):
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
progress = PipelineProgress(ctx) progress = PipelineProgress(ctx)
# Initialize Store for backend resolution # Initialize command-scope dependency context (caches Store/plugins)
try: deps = _CommandDependencies(config)
storage_registry = Store(config) storage_registry = deps.get_store()
except Exception:
storage_registry = None
path_arg = parsed.get("path") path_arg = parsed.get("path")
location = parsed.get("store") location = parsed.get("store")
@@ -348,7 +393,7 @@ class Add_File(Cmdlet):
is_storage_backend_location = False is_storage_backend_location = False
if location: if location:
try: try:
store_for_lookup = storage_registry or Store(config) store_for_lookup = storage_registry or deps.get_store()
is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None
except Exception: except Exception:
is_storage_backend_location = False is_storage_backend_location = False
@@ -368,6 +413,7 @@ class Add_File(Cmdlet):
plugin_instance, plugin_instance,
config, config,
store_instance=storage_registry, store_instance=storage_registry,
deps=deps,
) )
effective_storage_backend_name = plugin_storage_backend or ( effective_storage_backend_name = plugin_storage_backend or (
@@ -629,10 +675,11 @@ class Add_File(Cmdlet):
config, config,
export_destination=(Path(location) if location and not is_storage_backend_location else None), export_destination=(Path(location) if location and not is_storage_backend_location else None),
store_instance=storage_registry, store_instance=storage_registry,
deps=deps,
) )
if not media_path and plugin_name: if not media_path and plugin_name:
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source( media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source(
pipe_obj, config, storage_registry pipe_obj, config, storage_registry, deps=deps
) )
if media_path: if media_path:
try: try:
@@ -702,7 +749,7 @@ class Add_File(Cmdlet):
if location: if location:
try: try:
store = storage_registry or Store(config) store = storage_registry or deps.get_store()
resolved_backend = Add_File._resolve_backend_by_name(store, str(location)) resolved_backend = Add_File._resolve_backend_by_name(store, str(location))
if resolved_backend is not None: if resolved_backend is not None:
code = self._handle_storage_backend( code = self._handle_storage_backend(
@@ -833,7 +880,8 @@ class Add_File(Cmdlet):
Add_File._apply_pending_relationships( Add_File._apply_pending_relationships(
pending_relationship_pairs, pending_relationship_pairs,
config, config,
store_instance=storage_registry store_instance=storage_registry,
deps=deps
) )
except Exception: except Exception:
pass pass
@@ -1063,6 +1111,7 @@ class Add_File(Cmdlet):
config: Dict[str, config: Dict[str,
Any], Any],
store_instance: Optional[Store] = None, store_instance: Optional[Store] = None,
deps: Optional[_CommandDependencies] = None,
) -> None: ) -> None:
"""Persist relationships to backends that support relationships. """Persist relationships to backends that support relationships.
@@ -1071,8 +1120,11 @@ class Add_File(Cmdlet):
if not pending: if not pending:
return return
if deps is None:
deps = _CommandDependencies(config)
try: try:
store = store_instance if store_instance is not None else Store(config) store = store_instance if store_instance is not None else deps.get_store()
except Exception: except Exception:
return return
@@ -1343,6 +1395,7 @@ class Add_File(Cmdlet):
Any], Any],
export_destination: Optional[Path] = None, export_destination: Optional[Path] = None,
store_instance: Optional[Any] = None, store_instance: Optional[Any] = None,
deps: Optional[_CommandDependencies] = None,
) -> Tuple[Optional[Path], ) -> Tuple[Optional[Path],
Optional[str], Optional[str],
Optional[Path]]: Optional[Path]]:
@@ -1371,9 +1424,9 @@ class Add_File(Cmdlet):
if r_hash and r_store: if r_hash and r_store:
try: try:
store = store_instance if deps is None:
if not store: deps = _CommandDependencies(config)
store = Store(config) store = store_instance or deps.get_store()
backend = Add_File._resolve_backend_by_name(store, r_store) backend = Add_File._resolve_backend_by_name(store, r_store)
if backend is not None: if backend is not None:
@@ -1441,6 +1494,7 @@ class Add_File(Cmdlet):
result, result,
pipe_obj, pipe_obj,
config, config,
deps=deps,
) )
if downloaded_path: if downloaded_path:
pipe_obj.path = str(downloaded_path) pipe_obj.path = str(downloaded_path)
@@ -1471,14 +1525,16 @@ class Add_File(Cmdlet):
config: Dict[str, Any], config: Dict[str, Any],
*, *,
store_instance: Optional[Any] = None, store_instance: Optional[Any] = None,
deps: Optional[_CommandDependencies] = None,
) -> Optional[str]: ) -> Optional[str]:
plugin_key = Add_File._normalize_provider_key(plugin_name) plugin_key = Add_File._normalize_provider_key(plugin_name)
if not plugin_key: if not plugin_key:
return None return None
from ProviderCore.registry import get_plugin_with_capability if deps is None:
deps = _CommandDependencies(config)
file_provider = get_plugin_with_capability(plugin_key, "upload", config) file_provider = deps.get_plugin_with_capability(plugin_key, "upload")
if file_provider is None: if file_provider is None:
return None return None
@@ -1528,6 +1584,7 @@ class Add_File(Cmdlet):
result: Any, result: Any,
pipe_obj: models.PipeObject, pipe_obj: models.PipeObject,
config: Dict[str, Any], config: Dict[str, Any],
deps: Optional[_CommandDependencies] = None,
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
plugin_key = None plugin_key = None
for source in ( for source in (
@@ -1544,9 +1601,10 @@ class Add_File(Cmdlet):
if not plugin_key: if not plugin_key:
return None, None, None return None, None, None
from ProviderCore.registry import get_plugin if deps is None:
deps = _CommandDependencies(config)
plugin = get_plugin(plugin_key, config) plugin = deps.get_plugin(plugin_key)
if plugin is None: if plugin is None:
return None, None, None return None, None, None
@@ -1562,16 +1620,17 @@ class Add_File(Cmdlet):
pipe_obj: models.PipeObject, pipe_obj: models.PipeObject,
config: Dict[str, Any], config: Dict[str, Any],
store_instance: Optional[Any], store_instance: Optional[Any],
deps: Optional[_CommandDependencies] = None,
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip() r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
r_store = str(getattr(pipe_obj, "store", None) or "").strip() r_store = str(getattr(pipe_obj, "store", None) or "").strip()
if not (r_hash and r_store): if not (r_hash and r_store):
return None, None, None return None, None, None
try: if deps is None:
store = store_instance or Store(config) deps = _CommandDependencies(config)
except Exception:
store = None store = store_instance or deps.get_store()
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None
if backend is None: if backend is None:
return None, None, None return None, None, None
@@ -2244,6 +2303,7 @@ class Add_File(Cmdlet):
relationships = Add_File._get_relationships(result, pipe_obj) relationships = Add_File._get_relationships(result, pipe_obj)
try: try:
write_sidecar(target_path, tags, url, f_hash) write_sidecar(target_path, tags, url, f_hash)
from SYS.metadata import write_metadata # lazy: avoids 1000+ module chain at startup
write_metadata( write_metadata(
target_path, target_path,
hash_value=f_hash, hash_value=f_hash,
+3 -1
View File
@@ -24,7 +24,8 @@ from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table from SYS.result_table import Table
from SYS.rich_display import stderr_console as get_stderr_console from SYS.rich_display import stderr_console as get_stderr_console
from SYS import pipeline as pipeline_context from SYS import pipeline as pipeline_context
from SYS.metadata import normalize_urls as normalize_url_list # SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid
# pulling in Cryptodome (~900ms) at module import time.
from SYS.selection_builder import ( from SYS.selection_builder import (
extract_selection_fields, extract_selection_fields,
extract_urls_from_selection_args, extract_urls_from_selection_args,
@@ -1226,6 +1227,7 @@ class Download_File(Cmdlet):
and not a.startswith("-") and not a.startswith("-")
) )
] ]
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
raw_url = normalize_url_list(url_candidates) raw_url = normalize_url_list(url_candidates)
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
+22 -25
View File
@@ -14,14 +14,20 @@ import sys
from SYS.logger import log, debug from SYS.logger import log, debug
from plugins.metadata_provider import ( # plugins.metadata_provider is deferred: it transitively loads yt_dlp, Cryptodome,
get_default_subject_scrape_plugin, # imdbinfo, musicbrainzngs and ~1400 modules (~1.5s). Import lazily on first use.
get_metadata_plugin, _METADATA_PROVIDER_MOD: Optional[Any] = None
get_metadata_plugin_for_url,
list_metadata_plugins,
scrape_isbn_metadata, def _mp() -> Any:
scrape_openlibrary_metadata, """Return the (lazily imported) plugins.metadata_provider module."""
) global _METADATA_PROVIDER_MOD
if _METADATA_PROVIDER_MOD is None:
import plugins.metadata_provider as _m
_METADATA_PROVIDER_MOD = _m
return _METADATA_PROVIDER_MOD
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Tuple from typing import Any, Dict, List, Optional, Sequence, Tuple
@@ -41,11 +47,6 @@ CmdletArg = sh.CmdletArg
SharedArgs = sh.SharedArgs SharedArgs = sh.SharedArgs
parse_cmdlet_args = sh.parse_cmdlet_args parse_cmdlet_args = sh.parse_cmdlet_args
try:
from SYS.metadata import extract_title
except ImportError:
extract_title = None
def _dedup_tags_preserve_order(tags: List[str]) -> List[str]: def _dedup_tags_preserve_order(tags: List[str]) -> List[str]:
"""Deduplicate tags case-insensitively while preserving order.""" """Deduplicate tags case-insensitively while preserving order."""
@@ -210,7 +211,7 @@ def _extract_tag_value(tags_list: List[str], namespace: str) -> Optional[str]:
def _scrape_openlibrary_metadata(olid: str) -> List[str]: def _scrape_openlibrary_metadata(olid: str) -> List[str]:
try: try:
return list(scrape_openlibrary_metadata(olid)) return list(_mp().scrape_openlibrary_metadata(olid))
except Exception as e: except Exception as e:
log(f"OpenLibrary scraping error: {e}", file=sys.stderr) log(f"OpenLibrary scraping error: {e}", file=sys.stderr)
return [] return []
@@ -218,7 +219,7 @@ def _scrape_openlibrary_metadata(olid: str) -> List[str]:
def _scrape_isbn_metadata(isbn: str) -> List[str]: def _scrape_isbn_metadata(isbn: str) -> List[str]:
try: try:
return list(scrape_isbn_metadata(isbn)) return list(_mp().scrape_isbn_metadata(isbn))
except Exception as e: except Exception as e:
log(f"ISBN scraping error: {e}", file=sys.stderr) log(f"ISBN scraping error: {e}", file=sys.stderr)
return [] return []
@@ -400,7 +401,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else "" scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
plugin = None plugin = None
if scrape_target.startswith(("http://", "https://")): if scrape_target.startswith(("http://", "https://")):
plugin = get_metadata_plugin_for_url(scrape_target, config) plugin = _mp().get_metadata_plugin_for_url(scrape_target, config)
if plugin is None: if plugin is None:
log("No metadata plugin can scrape this URL", file=sys.stderr) log("No metadata plugin can scrape this URL", file=sys.stderr)
return 1 return 1
@@ -412,9 +413,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
return 0 return 0
if scrape_target: if scrape_target:
plugin = get_metadata_plugin(scrape_target, config) plugin = _mp().get_metadata_plugin(scrape_target, config)
else: else:
plugin = get_default_subject_scrape_plugin(config) plugin = _mp().get_default_subject_scrape_plugin(config)
if plugin is None: if plugin is None:
if scrape_target: if scrape_target:
log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr) log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
@@ -749,7 +750,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
) )
return 0 return 0
plugin_for_apply = get_metadata_plugin(str(result_provider), config) plugin_for_apply = _mp().get_metadata_plugin(str(result_provider), config)
if plugin_for_apply is not None: if plugin_for_apply is not None:
apply_tags = plugin_for_apply.filter_tags_for_store_apply( apply_tags = plugin_for_apply.filter_tags_for_store_apply(
[str(t) for t in result_tags if t is not None] [str(t) for t in result_tags if t is not None]
@@ -944,18 +945,14 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
return 0 return 0
_SCRAPE_CHOICES = [] _SCRAPE_CHOICES = [
try:
_SCRAPE_CHOICES = sorted(list_metadata_plugins().keys())
except Exception:
_SCRAPE_CHOICES = [
"itunes", "itunes",
"openlibrary", "openlibrary",
"googlebooks", "googlebooks",
"google", "google",
"musicbrainz", "musicbrainz",
"imdb", "imdb",
] ]
class Get_Tag(Cmdlet): class Get_Tag(Cmdlet):
+23 -23
View File
@@ -40,32 +40,16 @@ except ImportError:
PdfWriter = None PdfWriter = None
PdfReader = None PdfReader = None
try: # Stub fallbacks used before SYS.metadata is lazily imported (or if unavailable).
from SYS.metadata import ( HAS_METADATA_API: bool = False
read_tags_from_file, _metadata_loaded: bool = False
merge_multiple_tag_lists,
)
HAS_METADATA_API = True
except ImportError:
HAS_METADATA_API = False
def read_tags_from_file(file_path: Path) -> List[str]: def read_tags_from_file(file_path: Path) -> List[str]:
return [] return []
def write_tags_to_file(
file_path: Path,
tags: List[str],
source_hashes: Optional[List[str]] = None,
url: Optional[List[str]] = None,
append: bool = False,
) -> bool:
return False
def dedup_tags_by_namespace(tags: List[str]) -> List[str]: def merge_multiple_tag_lists(sources: List[List[str]],
return tags
def merge_multiple_tag_lists(sources: List[List[str]],
strategy: str = "first") -> List[str]: strategy: str = "first") -> List[str]:
out: List[str] = [] out: List[str] = []
seen: set[str] = set() seen: set[str] = set()
@@ -77,12 +61,28 @@ except ImportError:
seen.add(s) seen.add(s)
return out return out
def write_metadata(*_args: Any, **_kwargs: Any) -> None:
return None def _ensure_metadata_imports() -> None:
"""Lazily import SYS.metadata to avoid loading Cryptodome (~1s) at startup."""
global _metadata_loaded, HAS_METADATA_API, read_tags_from_file, merge_multiple_tag_lists
if _metadata_loaded:
return
_metadata_loaded = True
try:
from SYS.metadata import ( # type: ignore[assignment]
read_tags_from_file as _rtf,
merge_multiple_tag_lists as _mml,
)
read_tags_from_file = _rtf # type: ignore[assignment]
merge_multiple_tag_lists = _mml # type: ignore[assignment]
HAS_METADATA_API = True
except ImportError:
HAS_METADATA_API = False
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Merge multiple files into one.""" """Merge multiple files into one."""
_ensure_metadata_imports()
# Parse help # Parse help
if should_show_help(args): if should_show_help(args):
+3 -4
View File
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import Any from typing import Any
import httpx
from SYS.result_table import Table from SYS.result_table import Table
@@ -68,9 +66,10 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
code = int(getattr(response, "status_code", 0) or 0) code = int(getattr(response, "status_code", 0) or 0)
ok = 200 <= code < 500 ok = 200 <= code < 500
return ok, f"{url} (HTTP {code})" return ok, f"{url} (HTTP {code})"
except httpx.TimeoutException:
return False, f"{url} (timeout)"
except Exception as exc: except Exception as exc:
import httpx as _httpx
if isinstance(exc, _httpx.TimeoutException):
return False, f"{url} (timeout)"
return False, f"{url} ({type(exc).__name__})" return False, f"{url} ({type(exc).__name__})"
+4
View File
@@ -875,3 +875,7 @@ try:
except Exception: except Exception:
# best-effort registration # best-effort registration
pass pass
# Backward-compatible alias: tests and callers may import `plugins.matrix.cmdnat`.
from . import commands as cmdnat # noqa: E402
+6
View File
@@ -0,0 +1,6 @@
$tests = Get-ChildItem "$PSScriptRoot\..\tests\test_*.py" | Select-Object -ExpandProperty Name
foreach ($f in $tests) {
Write-Host "=== $f ===" -NoNewline
$out = & c:/Forgejo/Medios-Macina/.venv/Scripts/python.exe -m pytest "tests/$f" -q --tb=line 2>&1 | Select-Object -Last 2
Write-Host " $out"
}
+26 -3
View File
@@ -4,10 +4,11 @@ This package contains wrappers around external tools (e.g. yt-dlp) so cmdlets ca
common defaults (cookies, timeouts, format selectors) and users can override them via common defaults (cookies, timeouts, format selectors) and users can override them via
`config.conf`. `config.conf`.
""" """
from __future__ import annotations
from .ytdlp import YtDlpTool, YtDlpDefaults # Lazy-loaded to avoid pulling in yt_dlp, playwright, and their heavy transitive
from .playwright import PlaywrightTool, PlaywrightDefaults # dependencies (~12 s) at package import time. Each submodule is loaded only when
from .florencevision import FlorenceVisionTool, FlorenceVisionDefaults # a name from it is first accessed through this package namespace.
__all__ = [ __all__ = [
"YtDlpTool", "YtDlpTool",
@@ -17,3 +18,25 @@ __all__ = [
"FlorenceVisionTool", "FlorenceVisionTool",
"FlorenceVisionDefaults", "FlorenceVisionDefaults",
] ]
_MODULE_ATTRS = {
"YtDlpTool": ".ytdlp",
"YtDlpDefaults": ".ytdlp",
"PlaywrightTool": ".playwright",
"PlaywrightDefaults": ".playwright",
"FlorenceVisionTool": ".florencevision",
"FlorenceVisionDefaults": ".florencevision",
}
def __getattr__(name: str) -> object:
submod = _MODULE_ATTRS.get(name)
if submod is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from importlib import import_module
mod = import_module(submod, package=__name__)
obj = getattr(mod, name)
# Cache on this module so subsequent accesses bypass __getattr__.
globals()[name] = obj
return obj