updating and refactoring codebase for improved performance and maintainability
This commit is contained in:
+53
-8
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
|
import threading
|
||||||
|
|
||||||
from .HTTP import HTTPClient
|
from .HTTP import HTTPClient
|
||||||
|
|
||||||
@@ -16,6 +17,43 @@ class API:
|
|||||||
def __init__(self, base_url: str, timeout: float = 10.0) -> None:
|
def __init__(self, base_url: str, timeout: float = 10.0) -> None:
|
||||||
self.base_url = str(base_url or "").rstrip("/")
|
self.base_url = str(base_url or "").rstrip("/")
|
||||||
self.timeout = float(timeout)
|
self.timeout = float(timeout)
|
||||||
|
self._http_client: Optional[HTTPClient] = None
|
||||||
|
self._http_client_lock = threading.Lock()
|
||||||
|
|
||||||
|
def _get_http_client(self) -> HTTPClient:
|
||||||
|
"""Return a reusable opened HTTP client for this API instance."""
|
||||||
|
client = self._http_client
|
||||||
|
if client is not None and getattr(client, "_client", None) is not None:
|
||||||
|
return client
|
||||||
|
|
||||||
|
with self._http_client_lock:
|
||||||
|
client = self._http_client
|
||||||
|
if client is None:
|
||||||
|
client = HTTPClient(timeout=self.timeout)
|
||||||
|
self._http_client = client
|
||||||
|
if getattr(client, "_client", None) is None:
|
||||||
|
client.__enter__()
|
||||||
|
return client
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
client = self._http_client
|
||||||
|
if client is None:
|
||||||
|
return
|
||||||
|
with self._http_client_lock:
|
||||||
|
current = self._http_client
|
||||||
|
if current is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
current.__exit__(None, None, None)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._http_client = None
|
||||||
|
|
||||||
|
def __del__(self) -> None:
|
||||||
|
try:
|
||||||
|
self.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
def _get_json(
|
def _get_json(
|
||||||
self,
|
self,
|
||||||
@@ -25,10 +63,10 @@ class API:
|
|||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
url = f"{self.base_url}/{str(path or '').lstrip('/')}"
|
url = f"{self.base_url}/{str(path or '').lstrip('/')}"
|
||||||
try:
|
try:
|
||||||
with HTTPClient(timeout=self.timeout, headers=headers) as client:
|
client = self._get_http_client()
|
||||||
response = client.get(url, params=params, allow_redirects=True)
|
response = client.get(url, params=params, headers=headers, allow_redirects=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.json()
|
return response.json()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise ApiError(f"API request failed for {url}: {exc}") from exc
|
raise ApiError(f"API request failed for {url}: {exc}") from exc
|
||||||
|
|
||||||
@@ -41,9 +79,16 @@ class API:
|
|||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
url = f"{self.base_url}/{str(path or '').lstrip('/')}"
|
url = f"{self.base_url}/{str(path or '').lstrip('/')}"
|
||||||
try:
|
try:
|
||||||
with HTTPClient(timeout=self.timeout, headers=headers) as client:
|
client = self._get_http_client()
|
||||||
response = client.post(url, json=json_data, params=params, allow_redirects=True)
|
response = client.request(
|
||||||
response.raise_for_status()
|
"POST",
|
||||||
return response.json()
|
url,
|
||||||
|
json=json_data,
|
||||||
|
params=params,
|
||||||
|
headers=headers,
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise ApiError(f"API request failed for {url}: {exc}") from exc
|
raise ApiError(f"API request failed for {url}: {exc}") from exc
|
||||||
|
|||||||
@@ -27,6 +27,22 @@ from ProviderCore.base import Provider, SearchResult
|
|||||||
|
|
||||||
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
|
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
|
||||||
|
|
||||||
|
# Plugin instance cache keyed by (name, config_fingerprint)
|
||||||
|
_plugin_instance_cache: Dict[Tuple[str, str], Optional[Provider]] = {}
|
||||||
|
_plugin_cache_lock = __import__("threading").Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _config_fingerprint(config: Optional[Dict[str, Any]]) -> str:
|
||||||
|
"""Create a stable fingerprint of config for caching purposes."""
|
||||||
|
if config is None:
|
||||||
|
return "none"
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
normalized = json.dumps(config, sort_keys=True, default=str)
|
||||||
|
return hashlib.md5(normalized.encode()).hexdigest()[:16]
|
||||||
|
except Exception:
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
def _class_supports_method(
|
def _class_supports_method(
|
||||||
plugin_class: Type[Provider],
|
plugin_class: Type[Provider],
|
||||||
@@ -603,14 +619,26 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
|
|||||||
debug(f"[plugin] Unknown plugin: {name}")
|
debug(f"[plugin] Unknown plugin: {name}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# Check cache first
|
||||||
|
cache_key = (str(name).strip().lower(), _config_fingerprint(config))
|
||||||
|
with _plugin_cache_lock:
|
||||||
|
if cache_key in _plugin_instance_cache:
|
||||||
|
return _plugin_instance_cache[cache_key]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
plugin = info.plugin_class(config)
|
plugin = info.plugin_class(config)
|
||||||
if not plugin.validate():
|
if not plugin.validate():
|
||||||
debug(f"[plugin] Plugin '{name}' is not available")
|
debug(f"[plugin] Plugin '{name}' is not available")
|
||||||
|
with _plugin_cache_lock:
|
||||||
|
_plugin_instance_cache[cache_key] = None
|
||||||
return None
|
return None
|
||||||
|
with _plugin_cache_lock:
|
||||||
|
_plugin_instance_cache[cache_key] = plugin
|
||||||
return plugin
|
return plugin
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
debug(f"[plugin] Error initializing '{name}': {exc}")
|
debug(f"[plugin] Error initializing '{name}': {exc}")
|
||||||
|
with _plugin_cache_lock:
|
||||||
|
_plugin_instance_cache[cache_key] = None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -860,6 +888,13 @@ def resolve_inline_filters(
|
|||||||
return filters
|
return filters
|
||||||
|
|
||||||
|
|
||||||
|
def clear_plugin_cache() -> None:
|
||||||
|
"""Clear the plugin instance cache. Useful for testing or config reloads."""
|
||||||
|
global _plugin_instance_cache
|
||||||
|
with _plugin_cache_lock:
|
||||||
|
_plugin_instance_cache.clear()
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"PluginInfo",
|
"PluginInfo",
|
||||||
"Provider",
|
"Provider",
|
||||||
@@ -879,4 +914,5 @@ __all__ = [
|
|||||||
"selection_auto_stage_for_table",
|
"selection_auto_stage_for_table",
|
||||||
"plugin_inline_query_choices",
|
"plugin_inline_query_choices",
|
||||||
"is_known_plugin_name",
|
"is_known_plugin_name",
|
||||||
|
"clear_plugin_cache",
|
||||||
]
|
]
|
||||||
|
|||||||
+38
-11
@@ -10,18 +10,45 @@ import re
|
|||||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
||||||
from SYS.logger import debug
|
from SYS.logger import debug
|
||||||
|
|
||||||
# Prompt-toolkit lexer types are optional at import time; fall back to lightweight
|
# Prompt-toolkit lexer types are optional and expensive (~300ms). Use find_spec
|
||||||
# stubs if prompt_toolkit is not available so imports remain safe for testing.
|
# to detect availability without importing, then lazy-load on first use.
|
||||||
try:
|
import importlib.util as _importlib_util
|
||||||
from prompt_toolkit.document import Document
|
_PTK_AVAILABLE: bool = _importlib_util.find_spec("prompt_toolkit") is not None
|
||||||
from prompt_toolkit.lexers import Lexer as _PTK_Lexer
|
|
||||||
except Exception: # pragma: no cover - optional dependency
|
|
||||||
Document = object # type: ignore
|
|
||||||
# Fallback to a simple object when prompt_toolkit is not available
|
|
||||||
_PTK_Lexer = object # type: ignore
|
|
||||||
|
|
||||||
# Expose a stable name used by the rest of the module
|
_ptk_Document: Any = None
|
||||||
Lexer = _PTK_Lexer
|
_ptk_Lexer: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_ptk_Document() -> Any:
|
||||||
|
global _ptk_Document
|
||||||
|
if _ptk_Document is None:
|
||||||
|
if _PTK_AVAILABLE:
|
||||||
|
from prompt_toolkit.document import Document as _Doc
|
||||||
|
_ptk_Document = _Doc
|
||||||
|
else:
|
||||||
|
_ptk_Document = object
|
||||||
|
return _ptk_Document
|
||||||
|
|
||||||
|
|
||||||
|
def _get_ptk_Lexer() -> Any:
|
||||||
|
global _ptk_Lexer
|
||||||
|
if _ptk_Lexer is None:
|
||||||
|
if _PTK_AVAILABLE:
|
||||||
|
from prompt_toolkit.lexers import Lexer as _Lex
|
||||||
|
_ptk_Lexer = _Lex
|
||||||
|
else:
|
||||||
|
_ptk_Lexer = object
|
||||||
|
return _ptk_Lexer
|
||||||
|
|
||||||
|
|
||||||
|
# Stable aliases: these resolve lazily the first time they are accessed.
|
||||||
|
# Code that does `isinstance(x, Document)` or `class Foo(Lexer)` at class-body
|
||||||
|
# time needs the real object, so we keep module-level names that proxy to the
|
||||||
|
# lazy getters via __getattr__ on the module. Callers that reference
|
||||||
|
# Document/Lexer INSIDE functions will always get the real class.
|
||||||
|
# Populate the module-level names now so that class bodies below can inherit.
|
||||||
|
Document: Any = _get_ptk_Document()
|
||||||
|
Lexer: Any = _get_ptk_Lexer()
|
||||||
|
|
||||||
# Pre-compiled regexes for the lexer (avoid recompiling on every call)
|
# Pre-compiled regexes for the lexer (avoid recompiling on every call)
|
||||||
TOKEN_PATTERN = re.compile(
|
TOKEN_PATTERN = re.compile(
|
||||||
|
|||||||
+23
-12
@@ -1,13 +1,21 @@
|
|||||||
"""Unified logging utility for automatic file and function name tracking."""
|
"""Unified logging utility for automatic file and function name tracking."""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import inspect
|
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Optional, Sequence
|
from typing import Any, Optional, Sequence
|
||||||
|
|
||||||
from SYS.rich_display import console_for
|
# SYS.rich_display deferred: rich (~100ms) loaded lazily on first log output.
|
||||||
|
_rich_display_mod: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def _console_for(file):
|
||||||
|
global _rich_display_mod
|
||||||
|
if _rich_display_mod is None:
|
||||||
|
import SYS.rich_display as _m
|
||||||
|
_rich_display_mod = _m
|
||||||
|
return _rich_display_mod.console_for(file)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -73,7 +81,8 @@ def _is_rich_renderable(value: Any) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def _caller_location(depth: int = 1) -> tuple[str, str]:
|
def _caller_location(depth: int = 1) -> tuple[str, str]:
|
||||||
frame = inspect.currentframe()
|
import inspect as _inspect
|
||||||
|
frame = _inspect.currentframe()
|
||||||
current = frame
|
current = frame
|
||||||
try:
|
try:
|
||||||
for _ in range(max(0, int(depth))):
|
for _ in range(max(0, int(depth))):
|
||||||
@@ -160,7 +169,7 @@ def debug(*args, **kwargs) -> None:
|
|||||||
|
|
||||||
if len(args) == 1 and _is_rich_renderable(args[0]):
|
if len(args) == 1 and _is_rich_renderable(args[0]):
|
||||||
renderable = args[0]
|
renderable = args[0]
|
||||||
console_for(target_file).print(renderable)
|
_console_for(target_file).print(renderable)
|
||||||
file_name, func_name = _caller_location(depth=1)
|
file_name, func_name = _caller_location(depth=1)
|
||||||
caller_name = f"{file_name}.{func_name}" if file_name and func_name else ""
|
caller_name = f"{file_name}.{func_name}" if file_name and func_name else ""
|
||||||
_debug_db_log(caller_name=caller_name, message=f"<rich:{type(renderable).__name__}>")
|
_debug_db_log(caller_name=caller_name, message=f"<rich:{type(renderable).__name__}>")
|
||||||
@@ -200,7 +209,8 @@ def debug_inspect(
|
|||||||
|
|
||||||
# Compute caller prefix (same as log()).
|
# Compute caller prefix (same as log()).
|
||||||
prefix = None
|
prefix = None
|
||||||
frame = inspect.currentframe()
|
import inspect as _inspect
|
||||||
|
frame = _inspect.currentframe()
|
||||||
if frame is not None and frame.f_back is not None:
|
if frame is not None and frame.f_back is not None:
|
||||||
caller_frame = frame.f_back
|
caller_frame = frame.f_back
|
||||||
try:
|
try:
|
||||||
@@ -215,7 +225,7 @@ def debug_inspect(
|
|||||||
# Render.
|
# Render.
|
||||||
from rich import inspect as rich_inspect
|
from rich import inspect as rich_inspect
|
||||||
|
|
||||||
console = console_for(file)
|
console = _console_for(file)
|
||||||
# If the caller provides a title, treat it as authoritative.
|
# If the caller provides a title, treat it as authoritative.
|
||||||
# Only fall back to the automatic [file.func] prefix when no title is supplied.
|
# Only fall back to the automatic [file.func] prefix when no title is supplied.
|
||||||
effective_title = title
|
effective_title = title
|
||||||
@@ -266,12 +276,13 @@ def log(*args, **kwargs) -> None:
|
|||||||
add_prefix = _DEBUG_ENABLED
|
add_prefix = _DEBUG_ENABLED
|
||||||
|
|
||||||
# Get the calling frame
|
# Get the calling frame
|
||||||
frame = inspect.currentframe()
|
import inspect as _inspect
|
||||||
|
frame = _inspect.currentframe()
|
||||||
if frame is None:
|
if frame is None:
|
||||||
file = kwargs.pop("file", sys.stdout)
|
file = kwargs.pop("file", sys.stdout)
|
||||||
sep = kwargs.pop("sep", " ")
|
sep = kwargs.pop("sep", " ")
|
||||||
end = kwargs.pop("end", "\n")
|
end = kwargs.pop("end", "\n")
|
||||||
console_for(file).print(*args, sep=sep, end=end)
|
_console_for(file).print(*args, sep=sep, end=end)
|
||||||
return
|
return
|
||||||
|
|
||||||
caller_frame = frame.f_back
|
caller_frame = frame.f_back
|
||||||
@@ -279,7 +290,7 @@ def log(*args, **kwargs) -> None:
|
|||||||
file = kwargs.pop("file", sys.stdout)
|
file = kwargs.pop("file", sys.stdout)
|
||||||
sep = kwargs.pop("sep", " ")
|
sep = kwargs.pop("sep", " ")
|
||||||
end = kwargs.pop("end", "\n")
|
end = kwargs.pop("end", "\n")
|
||||||
console_for(file).print(*args, sep=sep, end=end)
|
_console_for(file).print(*args, sep=sep, end=end)
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -302,9 +313,9 @@ def log(*args, **kwargs) -> None:
|
|||||||
end = kwargs.pop("end", "\n")
|
end = kwargs.pop("end", "\n")
|
||||||
if add_prefix:
|
if add_prefix:
|
||||||
prefix = f"[{file_name}.{func_name}]"
|
prefix = f"[{file_name}.{func_name}]"
|
||||||
console_for(file).print(prefix, *args, sep=sep, end=end)
|
_console_for(file).print(prefix, *args, sep=sep, end=end)
|
||||||
else:
|
else:
|
||||||
console_for(file).print(*args, sep=sep, end=end)
|
_console_for(file).print(*args, sep=sep, end=end)
|
||||||
|
|
||||||
# Log to database if available
|
# Log to database if available
|
||||||
if _DB_LOGGER:
|
if _DB_LOGGER:
|
||||||
@@ -316,4 +327,4 @@ def log(*args, **kwargs) -> None:
|
|||||||
pass
|
pass
|
||||||
finally:
|
finally:
|
||||||
del frame
|
del frame
|
||||||
del caller_frame
|
del caller_frame
|
||||||
+80
-56
@@ -1,5 +1,7 @@
|
|||||||
"""Data models for the pipeline."""
|
"""Data models for the pipeline."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import hashlib
|
import hashlib
|
||||||
import inspect
|
import inspect
|
||||||
@@ -16,23 +18,9 @@ from dataclasses import dataclass, field
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Dict, List, Optional, Protocol, TextIO
|
from typing import Any, Callable, Dict, List, Optional, Protocol, TextIO
|
||||||
|
|
||||||
from rich.console import Console
|
# rich imports are deferred to avoid ~100ms startup cost.
|
||||||
from rich.console import ConsoleOptions
|
# Classes in this module that use rich types (ProgressBar, PipelineLiveProgress)
|
||||||
from rich.console import Group
|
# import them lazily inside their method bodies at first use.
|
||||||
from rich.live import Live
|
|
||||||
from rich.panel import Panel
|
|
||||||
from rich.progress import (
|
|
||||||
BarColumn,
|
|
||||||
DownloadColumn,
|
|
||||||
Progress,
|
|
||||||
SpinnerColumn,
|
|
||||||
TaskID,
|
|
||||||
TaskProgressColumn,
|
|
||||||
TextColumn,
|
|
||||||
TimeRemainingColumn,
|
|
||||||
TimeElapsedColumn,
|
|
||||||
TransferSpeedColumn,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -440,6 +428,42 @@ def _sanitise_for_json(
|
|||||||
return repr(value)
|
return repr(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _import_rich() -> Any:
|
||||||
|
"""Lazy-load rich types used by ProgressBar and PipelineLiveProgress."""
|
||||||
|
import rich.console as _rc
|
||||||
|
import rich.live as _rl
|
||||||
|
import rich.panel as _rp
|
||||||
|
import rich.progress as _rprog
|
||||||
|
# Return a namespace-like object with the types we need
|
||||||
|
class _Rich:
|
||||||
|
Console = _rc.Console
|
||||||
|
ConsoleOptions = _rc.ConsoleOptions
|
||||||
|
Group = _rc.Group
|
||||||
|
Live = _rl.Live
|
||||||
|
Panel = _rp.Panel
|
||||||
|
Progress = _rprog.Progress
|
||||||
|
BarColumn = _rprog.BarColumn
|
||||||
|
DownloadColumn = _rprog.DownloadColumn
|
||||||
|
SpinnerColumn = _rprog.SpinnerColumn
|
||||||
|
TaskID = _rprog.TaskID
|
||||||
|
TaskProgressColumn = _rprog.TaskProgressColumn
|
||||||
|
TextColumn = _rprog.TextColumn
|
||||||
|
TimeRemainingColumn = _rprog.TimeRemainingColumn
|
||||||
|
TimeElapsedColumn = _rprog.TimeElapsedColumn
|
||||||
|
TransferSpeedColumn = _rprog.TransferSpeedColumn
|
||||||
|
return _Rich
|
||||||
|
|
||||||
|
|
||||||
|
_rich: Any = None # cached after first call
|
||||||
|
|
||||||
|
|
||||||
|
def _r() -> Any:
|
||||||
|
global _rich
|
||||||
|
if _rich is None:
|
||||||
|
_rich = _import_rich()
|
||||||
|
return _rich
|
||||||
|
|
||||||
|
|
||||||
class ProgressBar:
|
class ProgressBar:
|
||||||
"""Rich progress helper for byte-based transfers.
|
"""Rich progress helper for byte-based transfers.
|
||||||
|
|
||||||
@@ -521,16 +545,16 @@ class ProgressBar:
|
|||||||
console = stderr_console()
|
console = stderr_console()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Failed to acquire shared stderr Console from SYS.rich_display; using fallback Console")
|
logger.exception("Failed to acquire shared stderr Console from SYS.rich_display; using fallback Console")
|
||||||
console = Console(file=stream)
|
console = _r().Console(file=stream)
|
||||||
else:
|
else:
|
||||||
console = Console(file=stream)
|
console = _r().Console(file=stream)
|
||||||
progress = Progress(
|
progress = _r().Progress(
|
||||||
TextColumn("[progress.description]{task.description}"),
|
_r().TextColumn("[progress.description]{task.description}"),
|
||||||
BarColumn(),
|
_r().BarColumn(),
|
||||||
TaskProgressColumn(),
|
_r().TaskProgressColumn(),
|
||||||
DownloadColumn(),
|
_r().DownloadColumn(),
|
||||||
TransferSpeedColumn(),
|
_r().TransferSpeedColumn(),
|
||||||
TimeRemainingColumn(),
|
_r().TimeRemainingColumn(),
|
||||||
console=console,
|
console=console,
|
||||||
transient=True,
|
transient=True,
|
||||||
)
|
)
|
||||||
@@ -867,7 +891,7 @@ class PipelineLiveProgress:
|
|||||||
overall = self._overall
|
overall = self._overall
|
||||||
if pipe_progress is None or transfers is None or overall is None:
|
if pipe_progress is None or transfers is None or overall is None:
|
||||||
# Not started (or stopped).
|
# Not started (or stopped).
|
||||||
yield Panel("", title="Pipeline", expand=False)
|
yield _r().Panel("", title="Pipeline", expand=False)
|
||||||
return
|
return
|
||||||
|
|
||||||
body_parts: List[Any] = [pipe_progress]
|
body_parts: List[Any] = [pipe_progress]
|
||||||
@@ -875,8 +899,8 @@ class PipelineLiveProgress:
|
|||||||
body_parts.append(status)
|
body_parts.append(status)
|
||||||
body_parts.append(transfers)
|
body_parts.append(transfers)
|
||||||
|
|
||||||
yield Group(
|
yield _r().Group(
|
||||||
Panel(Group(*body_parts),
|
_r().Panel(_r().Group(*body_parts),
|
||||||
title=self._title_text(),
|
title=self._title_text(),
|
||||||
expand=False),
|
expand=False),
|
||||||
overall
|
overall
|
||||||
@@ -895,8 +919,8 @@ class PipelineLiveProgress:
|
|||||||
if status is not None and self._status_tasks:
|
if status is not None and self._status_tasks:
|
||||||
body_parts.append(status)
|
body_parts.append(status)
|
||||||
body_parts.append(transfers)
|
body_parts.append(transfers)
|
||||||
return Group(
|
return _r().Group(
|
||||||
Panel(Group(*body_parts),
|
_r().Panel(_r().Group(*body_parts),
|
||||||
title=self._title_text(),
|
title=self._title_text(),
|
||||||
expand=False),
|
expand=False),
|
||||||
overall
|
overall
|
||||||
@@ -911,55 +935,55 @@ class PipelineLiveProgress:
|
|||||||
# IMPORTANT: use the shared stderr Console instance so that any
|
# IMPORTANT: use the shared stderr Console instance so that any
|
||||||
# `stderr_console().print(...)` calls from inside cmdlets (e.g. preflight
|
# `stderr_console().print(...)` calls from inside cmdlets (e.g. preflight
|
||||||
# tables/prompts in download-file) cooperate with Rich Live rendering.
|
# tables/prompts in download-file) cooperate with Rich Live rendering.
|
||||||
# If we create a separate Console(file=sys.stderr), output will fight for
|
# If we create a separate _r().Console(file=sys.stderr), output will fight for
|
||||||
# terminal cursor control and appear "blocked"/truncated.
|
# terminal cursor control and appear "blocked"/truncated.
|
||||||
from SYS.rich_display import stderr_console
|
from SYS.rich_display import stderr_console
|
||||||
|
|
||||||
self._console = stderr_console()
|
self._console = stderr_console()
|
||||||
|
|
||||||
# Persistent per-pipe bars.
|
# Persistent per-pipe bars.
|
||||||
self._pipe_progress = Progress(
|
self._pipe_progress = _r().Progress(
|
||||||
TextColumn("{task.description}"),
|
_r().TextColumn("{task.description}"),
|
||||||
TimeElapsedColumn(),
|
_r().TimeElapsedColumn(),
|
||||||
BarColumn(),
|
_r().BarColumn(),
|
||||||
TaskProgressColumn(),
|
_r().TaskProgressColumn(),
|
||||||
console=self._console,
|
console=self._console,
|
||||||
transient=False,
|
transient=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Transient, per-item spinner for the currently-active subtask.
|
# Transient, per-item spinner for the currently-active subtask.
|
||||||
self._subtasks = Progress(
|
self._subtasks = _r().Progress(
|
||||||
TextColumn(" "),
|
_r().TextColumn(" "),
|
||||||
SpinnerColumn("simpleDots"),
|
_r().SpinnerColumn("simpleDots"),
|
||||||
TextColumn("{task.description}"),
|
_r().TextColumn("{task.description}"),
|
||||||
console=self._console,
|
console=self._console,
|
||||||
transient=False,
|
transient=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Status line below the pipe bars. Kept simple (no extra bar) so it
|
# Status line below the pipe bars. Kept simple (no extra bar) so it
|
||||||
# doesn't visually offset the main pipe bar columns.
|
# doesn't visually offset the main pipe bar columns.
|
||||||
self._status = Progress(
|
self._status = _r().Progress(
|
||||||
TextColumn(" [bold]└─ {task.description}[/bold]"),
|
_r().TextColumn(" [bold]└─ {task.description}[/bold]"),
|
||||||
console=self._console,
|
console=self._console,
|
||||||
transient=False,
|
transient=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Byte-based transfer bars (download/upload) integrated into the Live view.
|
# Byte-based transfer bars (download/upload) integrated into the Live view.
|
||||||
self._transfers = Progress(
|
self._transfers = _r().Progress(
|
||||||
TextColumn(" {task.description}"),
|
_r().TextColumn(" {task.description}"),
|
||||||
BarColumn(),
|
_r().BarColumn(),
|
||||||
TaskProgressColumn(),
|
_r().TaskProgressColumn(),
|
||||||
DownloadColumn(),
|
_r().DownloadColumn(),
|
||||||
TransferSpeedColumn(),
|
_r().TransferSpeedColumn(),
|
||||||
TimeRemainingColumn(),
|
_r().TimeRemainingColumn(),
|
||||||
console=self._console,
|
console=self._console,
|
||||||
transient=False,
|
transient=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._overall = Progress(
|
self._overall = _r().Progress(
|
||||||
TimeElapsedColumn(),
|
_r().TimeElapsedColumn(),
|
||||||
BarColumn(),
|
_r().BarColumn(),
|
||||||
TextColumn("{task.description}"),
|
_r().TextColumn("{task.description}"),
|
||||||
console=self._console,
|
console=self._console,
|
||||||
transient=False,
|
transient=False,
|
||||||
)
|
)
|
||||||
@@ -982,7 +1006,7 @@ class PipelineLiveProgress:
|
|||||||
len(self._pipe_labels)),
|
len(self._pipe_labels)),
|
||||||
)
|
)
|
||||||
|
|
||||||
self._live = Live(
|
self._live = _r().Live(
|
||||||
self,
|
self,
|
||||||
console=self._console,
|
console=self._console,
|
||||||
refresh_per_second=10,
|
refresh_per_second=10,
|
||||||
@@ -1011,7 +1035,7 @@ class PipelineLiveProgress:
|
|||||||
# Not initialized yet; start fresh.
|
# Not initialized yet; start fresh.
|
||||||
self.start()
|
self.start()
|
||||||
return
|
return
|
||||||
self._live = Live(
|
self._live = _r().Live(
|
||||||
self,
|
self,
|
||||||
console=self._console,
|
console=self._console,
|
||||||
refresh_per_second=10,
|
refresh_per_second=10,
|
||||||
|
|||||||
+49
-35
@@ -13,11 +13,41 @@ from SYS.models import PipelineStageContext
|
|||||||
from SYS.logger import log, debug, debug_panel, is_debug_enabled
|
from SYS.logger import log, debug, debug_panel, is_debug_enabled
|
||||||
import logging
|
import logging
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
from SYS.worker import WorkerManagerRegistry, WorkerStages
|
# SYS.worker deferred: ffmpeg+attr+rich (~260ms) loaded lazily on first pipeline run.
|
||||||
from SYS.cli_parsing import SelectionSyntax, SelectionFilterSyntax
|
_worker_mod: Any = None
|
||||||
from SYS.rich_display import stdout_console
|
# SYS.cli_parsing deferred: prompt_toolkit (~300ms) loaded lazily on first selection.
|
||||||
from SYS.background_notifier import ensure_background_notifier
|
_cli_parsing_mod: Any = None
|
||||||
from SYS.result_table import Table
|
|
||||||
|
|
||||||
|
def _worker():
|
||||||
|
global _worker_mod
|
||||||
|
if _worker_mod is None:
|
||||||
|
import SYS.worker as _m
|
||||||
|
_worker_mod = _m
|
||||||
|
return _worker_mod
|
||||||
|
|
||||||
|
|
||||||
|
def _cli_parsing():
|
||||||
|
global _cli_parsing_mod
|
||||||
|
if _cli_parsing_mod is None:
|
||||||
|
import SYS.cli_parsing as _m
|
||||||
|
_cli_parsing_mod = _m
|
||||||
|
return _cli_parsing_mod
|
||||||
|
|
||||||
|
|
||||||
|
# SYS.rich_display deferred: rich (~100ms) loaded lazily on first console output.
|
||||||
|
# SYS.background_notifier deferred: rich/attr/ffmpeg loaded lazily on first notifier use.
|
||||||
|
# SYS.result_table deferred: textual (~140ms) loaded lazily on first Table use.
|
||||||
|
_result_table_mod: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def _result_table():
|
||||||
|
global _result_table_mod
|
||||||
|
if _result_table_mod is None:
|
||||||
|
from SYS.result_table import Table as _T
|
||||||
|
_result_table_mod = _T
|
||||||
|
return _result_table_mod
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from SYS.cmdlet_catalog import import_cmd_module
|
from SYS.cmdlet_catalog import import_cmd_module
|
||||||
@@ -680,12 +710,14 @@ def set_last_result_table(
|
|||||||
"""
|
"""
|
||||||
state = _get_pipeline_state()
|
state = _get_pipeline_state()
|
||||||
|
|
||||||
# Push current table to history before replacing
|
# Push current table to history before replacing.
|
||||||
|
# No .copy() needed: last_result_items is about to be replaced by reference,
|
||||||
|
# not mutated in place, so the old list reference is safe to keep in history.
|
||||||
if state.last_result_table is not None:
|
if state.last_result_table is not None:
|
||||||
state.result_table_history.append(
|
state.result_table_history.append(
|
||||||
(
|
(
|
||||||
state.last_result_table,
|
state.last_result_table,
|
||||||
state.last_result_items.copy(),
|
state.last_result_items,
|
||||||
state.last_result_subject,
|
state.last_result_subject,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -724,26 +756,6 @@ def set_last_result_table(
|
|||||||
logger.exception("Failed to sort result_table and reorder items")
|
logger.exception("Failed to sort result_table and reorder items")
|
||||||
|
|
||||||
|
|
||||||
if (
|
|
||||||
result_table is not None
|
|
||||||
and hasattr(result_table, "sort_by_title")
|
|
||||||
and not getattr(result_table, "preserve_order", False)
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
result_table.sort_by_title()
|
|
||||||
# Re-order items list to match the sorted table
|
|
||||||
if state.display_items and hasattr(result_table, "rows"):
|
|
||||||
sorted_items: List[Any] = []
|
|
||||||
for row in result_table.rows:
|
|
||||||
src_idx = getattr(row, "source_index", None)
|
|
||||||
if isinstance(src_idx, int) and 0 <= src_idx < len(state.display_items):
|
|
||||||
sorted_items.append(state.display_items[src_idx])
|
|
||||||
if len(sorted_items) == len(result_table.rows):
|
|
||||||
state.display_items = sorted_items
|
|
||||||
except Exception:
|
|
||||||
logger.exception("Failed to sort overlay result_table and reorder items")
|
|
||||||
|
|
||||||
|
|
||||||
def set_last_result_table_overlay(
|
def set_last_result_table_overlay(
|
||||||
result_table: Optional[Any],
|
result_table: Optional[Any],
|
||||||
items: Optional[List[Any]] = None,
|
items: Optional[List[Any]] = None,
|
||||||
@@ -1423,7 +1435,7 @@ class PipelineExecutor:
|
|||||||
new_first_stage: List[str] = []
|
new_first_stage: List[str] = []
|
||||||
for token in first_stage_tokens:
|
for token in first_stage_tokens:
|
||||||
if token.startswith("@"): # selection
|
if token.startswith("@"): # selection
|
||||||
selection = SelectionSyntax.parse(token)
|
selection = _cli_parsing().SelectionSyntax.parse(token)
|
||||||
if selection is not None:
|
if selection is not None:
|
||||||
first_stage_selection_indices = sorted(
|
first_stage_selection_indices = sorted(
|
||||||
[i - 1 for i in selection]
|
[i - 1 for i in selection]
|
||||||
@@ -1848,6 +1860,7 @@ class PipelineExecutor:
|
|||||||
}
|
}
|
||||||
if output_fn:
|
if output_fn:
|
||||||
kwargs["output"] = output_fn
|
kwargs["output"] = output_fn
|
||||||
|
from SYS.background_notifier import ensure_background_notifier
|
||||||
ensure_background_notifier(worker_manager, **kwargs)
|
ensure_background_notifier(worker_manager, **kwargs)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Failed to enable background notifier for session_worker_ids=%r", session_worker_ids)
|
logger.exception("Failed to enable background notifier for session_worker_ids=%r", session_worker_ids)
|
||||||
@@ -2633,9 +2646,9 @@ class PipelineExecutor:
|
|||||||
)
|
)
|
||||||
|
|
||||||
piped_result: Any = None
|
piped_result: Any = None
|
||||||
worker_manager = WorkerManagerRegistry.ensure(config)
|
worker_manager = _worker().WorkerManagerRegistry.ensure(config)
|
||||||
pipeline_text = " | ".join(" ".join(stage) for stage in stages)
|
pipeline_text = " | ".join(" ".join(stage) for stage in stages)
|
||||||
pipeline_session = WorkerStages.begin_pipeline(
|
pipeline_session = _worker().WorkerStages.begin_pipeline(
|
||||||
worker_manager,
|
worker_manager,
|
||||||
pipeline_text=pipeline_text,
|
pipeline_text=pipeline_text,
|
||||||
config=config
|
config=config
|
||||||
@@ -2790,8 +2803,8 @@ class PipelineExecutor:
|
|||||||
|
|
||||||
if cmd_name.startswith("@"): # selection stage
|
if cmd_name.startswith("@"): # selection stage
|
||||||
selection_token = raw_stage_name
|
selection_token = raw_stage_name
|
||||||
selection = SelectionSyntax.parse(selection_token)
|
selection = _cli_parsing().SelectionSyntax.parse(selection_token)
|
||||||
filter_spec = SelectionFilterSyntax.parse(selection_token)
|
filter_spec = _cli_parsing().SelectionFilterSyntax.parse(selection_token)
|
||||||
is_select_all = selection_token.strip() == "@*"
|
is_select_all = selection_token.strip() == "@*"
|
||||||
if selection is None and filter_spec is None and not is_select_all:
|
if selection is None and filter_spec is None and not is_select_all:
|
||||||
print(f"Invalid selection: {selection_token}\n")
|
print(f"Invalid selection: {selection_token}\n")
|
||||||
@@ -2849,7 +2862,7 @@ class PipelineExecutor:
|
|||||||
elif filter_spec is not None:
|
elif filter_spec is not None:
|
||||||
selected_indices = [
|
selected_indices = [
|
||||||
i for i, item in enumerate(items_list)
|
i for i, item in enumerate(items_list)
|
||||||
if SelectionFilterSyntax.matches(item, filter_spec)
|
if _cli_parsing().SelectionFilterSyntax.matches(item, filter_spec)
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
selected_indices = sorted(
|
selected_indices = sorted(
|
||||||
@@ -2894,7 +2907,7 @@ class PipelineExecutor:
|
|||||||
if base_table is not None and hasattr(base_table, "copy_with_title"):
|
if base_table is not None and hasattr(base_table, "copy_with_title"):
|
||||||
new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results")
|
new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results")
|
||||||
else:
|
else:
|
||||||
new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results")
|
new_table = _result_table()(getattr(base_table, "title", "") if base_table is not None else "Results")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if base_table is not None and getattr(base_table, "table", None):
|
if base_table is not None and getattr(base_table, "table", None):
|
||||||
@@ -2918,6 +2931,7 @@ class PipelineExecutor:
|
|||||||
logger.exception("Failed to set last_result_table_overlay for filter selection")
|
logger.exception("Failed to set last_result_table_overlay for filter selection")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
from SYS.rich_display import stdout_console
|
||||||
stdout_console().print()
|
stdout_console().print()
|
||||||
stdout_console().print(new_table)
|
stdout_console().print(new_table)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -3118,7 +3132,7 @@ class PipelineExecutor:
|
|||||||
pipe_idx = pipe_index_by_stage.get(stage_index)
|
pipe_idx = pipe_index_by_stage.get(stage_index)
|
||||||
|
|
||||||
overlay_table: Any | None = None
|
overlay_table: Any | None = None
|
||||||
session = WorkerStages.begin_stage(
|
session = _worker().WorkerStages.begin_stage(
|
||||||
worker_manager,
|
worker_manager,
|
||||||
cmd_name=cmd_name,
|
cmd_name=cmd_name,
|
||||||
stage_tokens=stage_tokens,
|
stage_tokens=stage_tokens,
|
||||||
|
|||||||
+71
-44
@@ -18,20 +18,47 @@ from pathlib import Path
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from rich.box import SIMPLE
|
# rich imports are deferred to avoid ~100ms startup cost.
|
||||||
from rich.console import Group
|
# All rich types are only needed inside method bodies, so we lazily import on first use.
|
||||||
from rich.panel import Panel
|
_rich_mod: Any = None
|
||||||
from rich.prompt import Prompt
|
|
||||||
from rich.table import Table as RichTable
|
|
||||||
from rich.text import Text
|
|
||||||
|
|
||||||
# Optional Textual imports - graceful fallback if not available
|
|
||||||
try:
|
|
||||||
from textual.widgets import Tree
|
|
||||||
|
|
||||||
TEXTUAL_AVAILABLE = True
|
def _rich():
|
||||||
except ImportError:
|
global _rich_mod
|
||||||
TEXTUAL_AVAILABLE = False
|
if _rich_mod is None:
|
||||||
|
import types as _types
|
||||||
|
_m = _types.SimpleNamespace()
|
||||||
|
from rich.box import SIMPLE as _SIMPLE
|
||||||
|
from rich.console import Group as _Group
|
||||||
|
from rich.panel import Panel as _Panel
|
||||||
|
from rich.prompt import Prompt as _Prompt
|
||||||
|
from rich.table import Table as _RichTable
|
||||||
|
from rich.text import Text as _Text
|
||||||
|
_m.SIMPLE = _SIMPLE
|
||||||
|
_m.Group = _Group
|
||||||
|
_m.Panel = _Panel
|
||||||
|
_m.Prompt = _Prompt
|
||||||
|
_m.RichTable = _RichTable
|
||||||
|
_m.Text = _Text
|
||||||
|
_rich_mod = _m
|
||||||
|
return _rich_mod
|
||||||
|
|
||||||
|
|
||||||
|
# Optional Textual imports - lazily loaded to avoid pulling in ~300ms of textual
|
||||||
|
# at import time when the TUI is not being used.
|
||||||
|
import importlib.util as _importlib_util
|
||||||
|
TEXTUAL_AVAILABLE: bool = _importlib_util.find_spec("textual") is not None
|
||||||
|
|
||||||
|
# Tree is populated lazily on first call to build_metadata_tree().
|
||||||
|
_textual_Tree: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_textual_Tree() -> Any:
|
||||||
|
global _textual_Tree
|
||||||
|
if _textual_Tree is None:
|
||||||
|
from textual.widgets import Tree as _Tree
|
||||||
|
_textual_Tree = _Tree
|
||||||
|
return _textual_Tree
|
||||||
|
|
||||||
|
|
||||||
# Import ResultModel from the API for typing; avoid runtime redefinition issues
|
# Import ResultModel from the API for typing; avoid runtime redefinition issues
|
||||||
@@ -1591,11 +1618,11 @@ class Table:
|
|||||||
panel_style = get_result_table_panel_style({"table_appearance": appearance_mode})
|
panel_style = get_result_table_panel_style({"table_appearance": appearance_mode})
|
||||||
|
|
||||||
if not self.rows:
|
if not self.rows:
|
||||||
empty = Text("No results")
|
empty = _rich().Text("No results")
|
||||||
return (
|
return (
|
||||||
Panel(
|
_rich().Panel(
|
||||||
empty,
|
empty,
|
||||||
title=Text(str(self.title), style=header_style),
|
title=_rich().Text(str(self.title), style=header_style),
|
||||||
border_style=border_style,
|
border_style=border_style,
|
||||||
padding=(0, 0),
|
padding=(0, 0),
|
||||||
expand=False,
|
expand=False,
|
||||||
@@ -1613,7 +1640,7 @@ class Table:
|
|||||||
seen.add(col.name)
|
seen.add(col.name)
|
||||||
col_names.append(col.name)
|
col_names.append(col.name)
|
||||||
|
|
||||||
table = RichTable(
|
table = _rich().RichTable(
|
||||||
show_header=True,
|
show_header=True,
|
||||||
header_style=header_style,
|
header_style=header_style,
|
||||||
border_style=border_style,
|
border_style=border_style,
|
||||||
@@ -1661,12 +1688,12 @@ class Table:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if self.title or self.header_lines:
|
if self.title or self.header_lines:
|
||||||
header_bits = [Text(line) for line in (self.header_lines or [])]
|
header_bits = [_rich().Text(line) for line in (self.header_lines or [])]
|
||||||
renderable = Group(*header_bits, table) if header_bits else table
|
renderable = _rich().Group(*header_bits, table) if header_bits else table
|
||||||
return (
|
return (
|
||||||
Panel(
|
_rich().Panel(
|
||||||
renderable,
|
renderable,
|
||||||
title=Text(str(self.title), style=header_style),
|
title=_rich().Text(str(self.title), style=header_style),
|
||||||
border_style=border_style,
|
border_style=border_style,
|
||||||
padding=(0, 0),
|
padding=(0, 0),
|
||||||
expand=False,
|
expand=False,
|
||||||
@@ -1777,7 +1804,7 @@ class Table:
|
|||||||
from SYS.rich_display import stdout_console
|
from SYS.rich_display import stdout_console
|
||||||
|
|
||||||
stdout_console().print(self)
|
stdout_console().print(self)
|
||||||
stdout_console().print(Panel(Text("Selection is disabled for this table.")))
|
stdout_console().print(_rich().Panel(_rich().Text("Selection is disabled for this table.")))
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Display the table
|
# Display the table
|
||||||
@@ -1789,11 +1816,11 @@ class Table:
|
|||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
if accept_args:
|
if accept_args:
|
||||||
choice = Prompt.ask(
|
choice = _rich().Prompt.ask(
|
||||||
f"{prompt} (e.g., '5' or '2 -storage hydrus' or 'q' to quit)"
|
f"{prompt} (e.g., '5' or '2 -storage hydrus' or 'q' to quit)"
|
||||||
).strip()
|
).strip()
|
||||||
else:
|
else:
|
||||||
choice = Prompt.ask(
|
choice = _rich().Prompt.ask(
|
||||||
f"{prompt} (e.g., '5' or '3-5' or '1,3,5' or 'q' to quit)"
|
f"{prompt} (e.g., '5' or '3-5' or '1,3,5' or 'q' to quit)"
|
||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
@@ -1806,8 +1833,8 @@ class Table:
|
|||||||
if result is not None:
|
if result is not None:
|
||||||
return result
|
return result
|
||||||
stdout_console().print(
|
stdout_console().print(
|
||||||
Panel(
|
_rich().Panel(
|
||||||
Text(
|
_rich().Text(
|
||||||
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
|
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1818,8 +1845,8 @@ class Table:
|
|||||||
if selected_indices is not None:
|
if selected_indices is not None:
|
||||||
return selected_indices
|
return selected_indices
|
||||||
stdout_console().print(
|
stdout_console().print(
|
||||||
Panel(
|
_rich().Panel(
|
||||||
Text(
|
_rich().Text(
|
||||||
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
|
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1827,16 +1854,16 @@ class Table:
|
|||||||
except (ValueError, EOFError):
|
except (ValueError, EOFError):
|
||||||
if accept_args:
|
if accept_args:
|
||||||
stdout_console().print(
|
stdout_console().print(
|
||||||
Panel(
|
_rich().Panel(
|
||||||
Text(
|
_rich().Text(
|
||||||
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
|
"Invalid format. Use: selection (5 or 3-5 or 1,3,5) optionally followed by flags (e.g., '5 -storage hydrus')."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
stdout_console().print(
|
stdout_console().print(
|
||||||
Panel(
|
_rich().Panel(
|
||||||
Text(
|
_rich().Text(
|
||||||
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
|
"Invalid format. Use: single (5), range (3-5), list (1,3,5), combined (1-3,7,9-11), or 'q' to quit."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -2468,12 +2495,12 @@ class ItemDetailView(Table):
|
|||||||
from rich.text import Text
|
from rich.text import Text
|
||||||
|
|
||||||
# 1. Create Detail Grid (matching rich_display.py style)
|
# 1. Create Detail Grid (matching rich_display.py style)
|
||||||
details_table = RichTable.grid(expand=True, padding=(0, 2))
|
details_table = _rich().RichTable.grid(expand=True, padding=(0, 2))
|
||||||
details_table.add_column("Key", style="cyan", justify="right", width=15)
|
details_table.add_column("Key", style="cyan", justify="right", width=15)
|
||||||
details_table.add_column("Value", style="white")
|
details_table.add_column("Value", style="white")
|
||||||
|
|
||||||
def _render_tag_text(tag_value: Any) -> Text:
|
def _render_tag_text(tag_value: Any) -> Text:
|
||||||
tag_text = Text()
|
tag_text = _rich().Text()
|
||||||
tag_text.append("#", style="dim")
|
tag_text.append("#", style="dim")
|
||||||
|
|
||||||
raw = str(tag_value or "")
|
raw = str(tag_value or "")
|
||||||
@@ -2497,17 +2524,17 @@ class ItemDetailView(Table):
|
|||||||
renderables.append(_render_tag_text(tag))
|
renderables.append(_render_tag_text(tag))
|
||||||
|
|
||||||
if freeform_tags:
|
if freeform_tags:
|
||||||
freeform_grid = RichTable.grid(expand=True, padding=(0, 2))
|
freeform_grid = _rich().RichTable.grid(expand=True, padding=(0, 2))
|
||||||
for _ in range(3):
|
for _ in range(3):
|
||||||
freeform_grid.add_column(ratio=1)
|
freeform_grid.add_column(ratio=1)
|
||||||
for row_values in _chunk_detail_tags(freeform_tags, 3):
|
for row_values in _chunk_detail_tags(freeform_tags, 3):
|
||||||
cells = [_render_tag_text(tag) for tag in row_values]
|
cells = [_render_tag_text(tag) for tag in row_values]
|
||||||
while len(cells) < 3:
|
while len(cells) < 3:
|
||||||
cells.append(Text(""))
|
cells.append(_rich().Text(""))
|
||||||
freeform_grid.add_row(*cells)
|
freeform_grid.add_row(*cells)
|
||||||
renderables.append(freeform_grid)
|
renderables.append(freeform_grid)
|
||||||
|
|
||||||
return Group(*renderables)
|
return _rich().Group(*renderables)
|
||||||
|
|
||||||
def _has_renderable_value(value: Any) -> bool:
|
def _has_renderable_value(value: Any) -> bool:
|
||||||
if value is None:
|
if value is None:
|
||||||
@@ -2596,9 +2623,9 @@ class ItemDetailView(Table):
|
|||||||
header_style = get_result_table_header_style()
|
header_style = get_result_table_header_style()
|
||||||
border_style = get_result_table_border_style()
|
border_style = get_result_table_border_style()
|
||||||
detail_title = str(self.detail_title or "Item Details").strip() or "Item Details"
|
detail_title = str(self.detail_title or "Item Details").strip() or "Item Details"
|
||||||
elements.append(Panel(
|
elements.append(_rich().Panel(
|
||||||
details_table,
|
details_table,
|
||||||
title=Text(detail_title, style=header_style),
|
title=_rich().Text(detail_title, style=header_style),
|
||||||
border_style=border_style,
|
border_style=border_style,
|
||||||
padding=(1, 2)
|
padding=(1, 2)
|
||||||
))
|
))
|
||||||
@@ -2606,10 +2633,10 @@ class ItemDetailView(Table):
|
|||||||
if results_renderable:
|
if results_renderable:
|
||||||
# If it's a Panel already (from super().to_rich() with title), use it directly
|
# If it's a Panel already (from super().to_rich() with title), use it directly
|
||||||
# but force the border style to the result-table standard for consistency
|
# but force the border style to the result-table standard for consistency
|
||||||
if isinstance(results_renderable, Panel):
|
if isinstance(results_renderable, _rich().Panel):
|
||||||
results_renderable.border_style = get_result_table_border_style()
|
results_renderable.border_style = get_result_table_border_style()
|
||||||
if results_renderable.title:
|
if results_renderable.title:
|
||||||
results_renderable.title = Text(
|
results_renderable.title = _rich().Text(
|
||||||
str(results_renderable.title),
|
str(results_renderable.title),
|
||||||
style=get_result_table_header_style(),
|
style=get_result_table_header_style(),
|
||||||
)
|
)
|
||||||
@@ -2622,13 +2649,13 @@ class ItemDetailView(Table):
|
|||||||
display_title = original_title
|
display_title = original_title
|
||||||
|
|
||||||
# Add a bit of padding
|
# Add a bit of padding
|
||||||
results_group = Group(Text(""), results_renderable, Text(""))
|
results_group = _rich().Group(_rich().Text(""), results_renderable, _rich().Text(""))
|
||||||
elements.append(
|
elements.append(
|
||||||
Panel(
|
_rich().Panel(
|
||||||
results_group,
|
results_group,
|
||||||
title=Text(str(display_title), style=get_result_table_header_style()),
|
title=_rich().Text(str(display_title), style=get_result_table_header_style()),
|
||||||
border_style=get_result_table_border_style(),
|
border_style=get_result_table_border_style(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return Group(*elements)
|
return _rich().Group(*elements)
|
||||||
|
|||||||
+30
-21
@@ -13,42 +13,49 @@ import contextlib
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Iterator, TextIO, List, Dict, Optional, Tuple, cast
|
from typing import Any, Iterator, TextIO, List, Dict, Optional, Tuple, cast
|
||||||
|
|
||||||
from rich.console import Console
|
|
||||||
from rich.panel import Panel
|
|
||||||
from rich.text import Text
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from SYS.utils import expand_path
|
from SYS.utils import expand_path
|
||||||
|
|
||||||
# Configure Rich pretty-printing to avoid truncating long strings (hashes/paths).
|
# rich imports are deferred to first Console use to avoid ~100ms startup cost.
|
||||||
# This is version-safe: older Rich versions may not support the max_* arguments.
|
# They are loaded the first time any Console function is called.
|
||||||
try:
|
|
||||||
from rich.pretty import install as _pretty_install
|
|
||||||
|
|
||||||
try:
|
_STDOUT_CONSOLE: Any = None
|
||||||
_pretty_install(max_string=100_000, max_length=100_000)
|
_STDERR_CONSOLE: Any = None
|
||||||
except TypeError:
|
|
||||||
_pretty_install()
|
|
||||||
except Exception:
|
|
||||||
from SYS.logger import logger
|
|
||||||
logger.exception("Failed to configure rich pretty-printing")
|
|
||||||
|
|
||||||
_STDOUT_CONSOLE = Console(file=sys.stdout)
|
|
||||||
_STDERR_CONSOLE = Console(file=sys.stderr)
|
|
||||||
|
|
||||||
|
|
||||||
def stdout_console() -> Console:
|
def _ensure_consoles() -> None:
|
||||||
|
global _STDOUT_CONSOLE, _STDERR_CONSOLE
|
||||||
|
if _STDOUT_CONSOLE is None:
|
||||||
|
from rich.console import Console
|
||||||
|
from rich.pretty import install as _pretty_install
|
||||||
|
try:
|
||||||
|
_pretty_install(max_string=100_000, max_length=100_000)
|
||||||
|
except TypeError:
|
||||||
|
_pretty_install()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
_STDOUT_CONSOLE = Console(file=sys.stdout)
|
||||||
|
_STDERR_CONSOLE = Console(file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def stdout_console() -> Any:
|
||||||
|
_ensure_consoles()
|
||||||
return _STDOUT_CONSOLE
|
return _STDOUT_CONSOLE
|
||||||
|
|
||||||
|
|
||||||
def stderr_console() -> Console:
|
def stderr_console() -> Any:
|
||||||
|
_ensure_consoles()
|
||||||
return _STDERR_CONSOLE
|
return _STDERR_CONSOLE
|
||||||
|
|
||||||
|
|
||||||
def console_for(file: TextIO | None) -> Console:
|
def console_for(file: Any) -> Any:
|
||||||
if file is None or file is sys.stdout:
|
if file is None or file is sys.stdout:
|
||||||
|
_ensure_consoles()
|
||||||
return _STDOUT_CONSOLE
|
return _STDOUT_CONSOLE
|
||||||
if file is sys.stderr:
|
if file is sys.stderr:
|
||||||
|
_ensure_consoles()
|
||||||
return _STDERR_CONSOLE
|
return _STDERR_CONSOLE
|
||||||
|
from rich.console import Console
|
||||||
return Console(file=file)
|
return Console(file=file)
|
||||||
|
|
||||||
|
|
||||||
@@ -57,7 +64,7 @@ def rprint(renderable: Any = "", *, file: TextIO | None = None) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]:
|
def capture_rich_output(*, stdout: Any, stderr: Any) -> Iterator[None]:
|
||||||
"""Temporarily redirect Rich output helpers to provided streams.
|
"""Temporarily redirect Rich output helpers to provided streams.
|
||||||
|
|
||||||
Note: `stdout_console()` / `stderr_console()` use global Console instances,
|
Note: `stdout_console()` / `stderr_console()` use global Console instances,
|
||||||
@@ -65,9 +72,11 @@ def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
global _STDOUT_CONSOLE, _STDERR_CONSOLE
|
global _STDOUT_CONSOLE, _STDERR_CONSOLE
|
||||||
|
_ensure_consoles()
|
||||||
|
|
||||||
previous_stdout = _STDOUT_CONSOLE
|
previous_stdout = _STDOUT_CONSOLE
|
||||||
previous_stderr = _STDERR_CONSOLE
|
previous_stderr = _STDERR_CONSOLE
|
||||||
|
from rich.console import Console
|
||||||
try:
|
try:
|
||||||
_STDOUT_CONSOLE = Console(file=stdout)
|
_STDOUT_CONSOLE = Console(file=stdout)
|
||||||
_STDERR_CONSOLE = Console(file=stderr)
|
_STDERR_CONSOLE = Console(file=stderr)
|
||||||
|
|||||||
+19
-7
@@ -4,13 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import hashlib
|
import hashlib
|
||||||
import subprocess
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
try:
|
|
||||||
import ffmpeg # type: ignore
|
|
||||||
except Exception:
|
|
||||||
ffmpeg = None # type: ignore
|
|
||||||
import os
|
import os
|
||||||
import base64
|
import base64
|
||||||
import logging
|
import logging
|
||||||
@@ -23,6 +17,22 @@ from urllib.parse import urlparse
|
|||||||
|
|
||||||
from SYS.utils_constant import mime_maps
|
from SYS.utils_constant import mime_maps
|
||||||
|
|
||||||
|
_ffmpeg_mod: Any = None
|
||||||
|
_ffmpeg_checked = False
|
||||||
|
|
||||||
|
|
||||||
|
def _get_ffmpeg():
|
||||||
|
"""Lazily return the ffmpeg module, or None if unavailable."""
|
||||||
|
global _ffmpeg_mod, _ffmpeg_checked
|
||||||
|
if not _ffmpeg_checked:
|
||||||
|
try:
|
||||||
|
import ffmpeg as _f # type: ignore
|
||||||
|
_ffmpeg_mod = _f
|
||||||
|
except Exception:
|
||||||
|
_ffmpeg_mod = None
|
||||||
|
_ffmpeg_checked = True
|
||||||
|
return _ffmpeg_mod
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import cbor2
|
import cbor2
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@@ -191,6 +201,7 @@ def ffprobe(file_path: str) -> dict:
|
|||||||
probe = None
|
probe = None
|
||||||
|
|
||||||
# Try python ffmpeg module first
|
# Try python ffmpeg module first
|
||||||
|
ffmpeg = _get_ffmpeg()
|
||||||
if ffmpeg is not None:
|
if ffmpeg is not None:
|
||||||
try:
|
try:
|
||||||
probe = ffmpeg.probe(file_path)
|
probe = ffmpeg.probe(file_path)
|
||||||
@@ -203,7 +214,8 @@ def ffprobe(file_path: str) -> dict:
|
|||||||
ffprobe_cmd = shutil.which("ffprobe")
|
ffprobe_cmd = shutil.which("ffprobe")
|
||||||
if ffprobe_cmd:
|
if ffprobe_cmd:
|
||||||
try:
|
try:
|
||||||
proc = subprocess.run(
|
import subprocess as _subprocess
|
||||||
|
proc = _subprocess.run(
|
||||||
[
|
[
|
||||||
ffprobe_cmd,
|
ffprobe_cmd,
|
||||||
"-v",
|
"-v",
|
||||||
|
|||||||
+82
-22
@@ -40,11 +40,58 @@ build_pipeline_preview = sh.build_pipeline_preview
|
|||||||
get_field = sh.get_field
|
get_field = sh.get_field
|
||||||
|
|
||||||
from SYS.utils import sha256_file, unique_path, sanitize_filename
|
from SYS.utils import sha256_file, unique_path, sanitize_filename
|
||||||
from SYS.metadata import write_metadata
|
|
||||||
|
|
||||||
# Canonical supported filetypes for all stores/cmdlets
|
# Canonical supported filetypes for all stores/cmdlets
|
||||||
SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS
|
SUPPORTED_MEDIA_EXTENSIONS = ALL_SUPPORTED_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
|
class _CommandDependencies:
|
||||||
|
"""Command-scope cache for Store and plugin instances to avoid repeated instantiation."""
|
||||||
|
|
||||||
|
def __init__(self, config: Dict[str, Any]) -> None:
|
||||||
|
self.config = config
|
||||||
|
self._store: Optional[Store] = None
|
||||||
|
self._plugins: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
def get_store(self) -> Optional[Store]:
|
||||||
|
"""Lazily initialize and return the command-scope Store instance."""
|
||||||
|
if self._store is None:
|
||||||
|
try:
|
||||||
|
self._store = Store(self.config)
|
||||||
|
except Exception:
|
||||||
|
self._store = None
|
||||||
|
return self._store
|
||||||
|
|
||||||
|
def get_plugin(self, name: str) -> Optional[Any]:
|
||||||
|
"""Cached plugin lookup by name."""
|
||||||
|
from ProviderCore.registry import get_plugin
|
||||||
|
|
||||||
|
norm_name = str(name or "").strip().lower()
|
||||||
|
if not norm_name:
|
||||||
|
return None
|
||||||
|
if norm_name in self._plugins:
|
||||||
|
return self._plugins[norm_name]
|
||||||
|
|
||||||
|
plugin = get_plugin(norm_name, self.config)
|
||||||
|
self._plugins[norm_name] = plugin
|
||||||
|
return plugin
|
||||||
|
|
||||||
|
def get_plugin_with_capability(self, name: str, capability: str) -> Optional[Any]:
|
||||||
|
"""Cached plugin lookup with capability check."""
|
||||||
|
from ProviderCore.registry import get_plugin_with_capability
|
||||||
|
|
||||||
|
norm_name = str(name or "").strip().lower()
|
||||||
|
if not norm_name:
|
||||||
|
return None
|
||||||
|
|
||||||
|
cache_key = f"{norm_name}#{capability}"
|
||||||
|
if cache_key in self._plugins:
|
||||||
|
return self._plugins[cache_key]
|
||||||
|
|
||||||
|
plugin = get_plugin_with_capability(norm_name, capability, self.config)
|
||||||
|
self._plugins[cache_key] = plugin
|
||||||
|
return plugin
|
||||||
|
|
||||||
DEBUG_PIPE_NOTE_PREVIEW_LENGTH = 256
|
DEBUG_PIPE_NOTE_PREVIEW_LENGTH = 256
|
||||||
|
|
||||||
# Protocol schemes that identify a remote resource / not a local file path.
|
# Protocol schemes that identify a remote resource / not a local file path.
|
||||||
@@ -220,11 +267,9 @@ class Add_File(Cmdlet):
|
|||||||
parsed = parse_cmdlet_args(args, self)
|
parsed = parse_cmdlet_args(args, self)
|
||||||
progress = PipelineProgress(ctx)
|
progress = PipelineProgress(ctx)
|
||||||
|
|
||||||
# Initialize Store for backend resolution
|
# Initialize command-scope dependency context (caches Store/plugins)
|
||||||
try:
|
deps = _CommandDependencies(config)
|
||||||
storage_registry = Store(config)
|
storage_registry = deps.get_store()
|
||||||
except Exception:
|
|
||||||
storage_registry = None
|
|
||||||
|
|
||||||
path_arg = parsed.get("path")
|
path_arg = parsed.get("path")
|
||||||
location = parsed.get("store")
|
location = parsed.get("store")
|
||||||
@@ -348,7 +393,7 @@ class Add_File(Cmdlet):
|
|||||||
is_storage_backend_location = False
|
is_storage_backend_location = False
|
||||||
if location:
|
if location:
|
||||||
try:
|
try:
|
||||||
store_for_lookup = storage_registry or Store(config)
|
store_for_lookup = storage_registry or deps.get_store()
|
||||||
is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None
|
is_storage_backend_location = Add_File._resolve_backend_by_name(store_for_lookup, str(location)) is not None
|
||||||
except Exception:
|
except Exception:
|
||||||
is_storage_backend_location = False
|
is_storage_backend_location = False
|
||||||
@@ -368,6 +413,7 @@ class Add_File(Cmdlet):
|
|||||||
plugin_instance,
|
plugin_instance,
|
||||||
config,
|
config,
|
||||||
store_instance=storage_registry,
|
store_instance=storage_registry,
|
||||||
|
deps=deps,
|
||||||
)
|
)
|
||||||
|
|
||||||
effective_storage_backend_name = plugin_storage_backend or (
|
effective_storage_backend_name = plugin_storage_backend or (
|
||||||
@@ -629,10 +675,11 @@ class Add_File(Cmdlet):
|
|||||||
config,
|
config,
|
||||||
export_destination=(Path(location) if location and not is_storage_backend_location else None),
|
export_destination=(Path(location) if location and not is_storage_backend_location else None),
|
||||||
store_instance=storage_registry,
|
store_instance=storage_registry,
|
||||||
|
deps=deps,
|
||||||
)
|
)
|
||||||
if not media_path and plugin_name:
|
if not media_path and plugin_name:
|
||||||
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source(
|
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source(
|
||||||
pipe_obj, config, storage_registry
|
pipe_obj, config, storage_registry, deps=deps
|
||||||
)
|
)
|
||||||
if media_path:
|
if media_path:
|
||||||
try:
|
try:
|
||||||
@@ -702,7 +749,7 @@ class Add_File(Cmdlet):
|
|||||||
|
|
||||||
if location:
|
if location:
|
||||||
try:
|
try:
|
||||||
store = storage_registry or Store(config)
|
store = storage_registry or deps.get_store()
|
||||||
resolved_backend = Add_File._resolve_backend_by_name(store, str(location))
|
resolved_backend = Add_File._resolve_backend_by_name(store, str(location))
|
||||||
if resolved_backend is not None:
|
if resolved_backend is not None:
|
||||||
code = self._handle_storage_backend(
|
code = self._handle_storage_backend(
|
||||||
@@ -833,7 +880,8 @@ class Add_File(Cmdlet):
|
|||||||
Add_File._apply_pending_relationships(
|
Add_File._apply_pending_relationships(
|
||||||
pending_relationship_pairs,
|
pending_relationship_pairs,
|
||||||
config,
|
config,
|
||||||
store_instance=storage_registry
|
store_instance=storage_registry,
|
||||||
|
deps=deps
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -1063,6 +1111,7 @@ class Add_File(Cmdlet):
|
|||||||
config: Dict[str,
|
config: Dict[str,
|
||||||
Any],
|
Any],
|
||||||
store_instance: Optional[Store] = None,
|
store_instance: Optional[Store] = None,
|
||||||
|
deps: Optional[_CommandDependencies] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Persist relationships to backends that support relationships.
|
"""Persist relationships to backends that support relationships.
|
||||||
|
|
||||||
@@ -1071,8 +1120,11 @@ class Add_File(Cmdlet):
|
|||||||
if not pending:
|
if not pending:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if deps is None:
|
||||||
|
deps = _CommandDependencies(config)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
store = store_instance if store_instance is not None else Store(config)
|
store = store_instance if store_instance is not None else deps.get_store()
|
||||||
except Exception:
|
except Exception:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -1343,6 +1395,7 @@ class Add_File(Cmdlet):
|
|||||||
Any],
|
Any],
|
||||||
export_destination: Optional[Path] = None,
|
export_destination: Optional[Path] = None,
|
||||||
store_instance: Optional[Any] = None,
|
store_instance: Optional[Any] = None,
|
||||||
|
deps: Optional[_CommandDependencies] = None,
|
||||||
) -> Tuple[Optional[Path],
|
) -> Tuple[Optional[Path],
|
||||||
Optional[str],
|
Optional[str],
|
||||||
Optional[Path]]:
|
Optional[Path]]:
|
||||||
@@ -1371,9 +1424,9 @@ class Add_File(Cmdlet):
|
|||||||
|
|
||||||
if r_hash and r_store:
|
if r_hash and r_store:
|
||||||
try:
|
try:
|
||||||
store = store_instance
|
if deps is None:
|
||||||
if not store:
|
deps = _CommandDependencies(config)
|
||||||
store = Store(config)
|
store = store_instance or deps.get_store()
|
||||||
|
|
||||||
backend = Add_File._resolve_backend_by_name(store, r_store)
|
backend = Add_File._resolve_backend_by_name(store, r_store)
|
||||||
if backend is not None:
|
if backend is not None:
|
||||||
@@ -1441,6 +1494,7 @@ class Add_File(Cmdlet):
|
|||||||
result,
|
result,
|
||||||
pipe_obj,
|
pipe_obj,
|
||||||
config,
|
config,
|
||||||
|
deps=deps,
|
||||||
)
|
)
|
||||||
if downloaded_path:
|
if downloaded_path:
|
||||||
pipe_obj.path = str(downloaded_path)
|
pipe_obj.path = str(downloaded_path)
|
||||||
@@ -1471,14 +1525,16 @@ class Add_File(Cmdlet):
|
|||||||
config: Dict[str, Any],
|
config: Dict[str, Any],
|
||||||
*,
|
*,
|
||||||
store_instance: Optional[Any] = None,
|
store_instance: Optional[Any] = None,
|
||||||
|
deps: Optional[_CommandDependencies] = None,
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
plugin_key = Add_File._normalize_provider_key(plugin_name)
|
plugin_key = Add_File._normalize_provider_key(plugin_name)
|
||||||
if not plugin_key:
|
if not plugin_key:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
from ProviderCore.registry import get_plugin_with_capability
|
if deps is None:
|
||||||
|
deps = _CommandDependencies(config)
|
||||||
|
|
||||||
file_provider = get_plugin_with_capability(plugin_key, "upload", config)
|
file_provider = deps.get_plugin_with_capability(plugin_key, "upload")
|
||||||
if file_provider is None:
|
if file_provider is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -1528,6 +1584,7 @@ class Add_File(Cmdlet):
|
|||||||
result: Any,
|
result: Any,
|
||||||
pipe_obj: models.PipeObject,
|
pipe_obj: models.PipeObject,
|
||||||
config: Dict[str, Any],
|
config: Dict[str, Any],
|
||||||
|
deps: Optional[_CommandDependencies] = None,
|
||||||
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||||
plugin_key = None
|
plugin_key = None
|
||||||
for source in (
|
for source in (
|
||||||
@@ -1544,9 +1601,10 @@ class Add_File(Cmdlet):
|
|||||||
if not plugin_key:
|
if not plugin_key:
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
from ProviderCore.registry import get_plugin
|
if deps is None:
|
||||||
|
deps = _CommandDependencies(config)
|
||||||
|
|
||||||
plugin = get_plugin(plugin_key, config)
|
plugin = deps.get_plugin(plugin_key)
|
||||||
if plugin is None:
|
if plugin is None:
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
@@ -1562,16 +1620,17 @@ class Add_File(Cmdlet):
|
|||||||
pipe_obj: models.PipeObject,
|
pipe_obj: models.PipeObject,
|
||||||
config: Dict[str, Any],
|
config: Dict[str, Any],
|
||||||
store_instance: Optional[Any],
|
store_instance: Optional[Any],
|
||||||
|
deps: Optional[_CommandDependencies] = None,
|
||||||
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||||
r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
|
r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
|
||||||
r_store = str(getattr(pipe_obj, "store", None) or "").strip()
|
r_store = str(getattr(pipe_obj, "store", None) or "").strip()
|
||||||
if not (r_hash and r_store):
|
if not (r_hash and r_store):
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
try:
|
if deps is None:
|
||||||
store = store_instance or Store(config)
|
deps = _CommandDependencies(config)
|
||||||
except Exception:
|
|
||||||
store = None
|
store = store_instance or deps.get_store()
|
||||||
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None
|
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None
|
||||||
if backend is None:
|
if backend is None:
|
||||||
return None, None, None
|
return None, None, None
|
||||||
@@ -2244,6 +2303,7 @@ class Add_File(Cmdlet):
|
|||||||
relationships = Add_File._get_relationships(result, pipe_obj)
|
relationships = Add_File._get_relationships(result, pipe_obj)
|
||||||
try:
|
try:
|
||||||
write_sidecar(target_path, tags, url, f_hash)
|
write_sidecar(target_path, tags, url, f_hash)
|
||||||
|
from SYS.metadata import write_metadata # lazy: avoids 1000+ module chain at startup
|
||||||
write_metadata(
|
write_metadata(
|
||||||
target_path,
|
target_path,
|
||||||
hash_value=f_hash,
|
hash_value=f_hash,
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ from SYS.pipeline_progress import PipelineProgress
|
|||||||
from SYS.result_table import Table
|
from SYS.result_table import Table
|
||||||
from SYS.rich_display import stderr_console as get_stderr_console
|
from SYS.rich_display import stderr_console as get_stderr_console
|
||||||
from SYS import pipeline as pipeline_context
|
from SYS import pipeline as pipeline_context
|
||||||
from SYS.metadata import normalize_urls as normalize_url_list
|
# SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid
|
||||||
|
# pulling in Cryptodome (~900ms) at module import time.
|
||||||
from SYS.selection_builder import (
|
from SYS.selection_builder import (
|
||||||
extract_selection_fields,
|
extract_selection_fields,
|
||||||
extract_urls_from_selection_args,
|
extract_urls_from_selection_args,
|
||||||
@@ -1226,6 +1227,7 @@ class Download_File(Cmdlet):
|
|||||||
and not a.startswith("-")
|
and not a.startswith("-")
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
|
||||||
raw_url = normalize_url_list(url_candidates)
|
raw_url = normalize_url_list(url_candidates)
|
||||||
|
|
||||||
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
|
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
|
||||||
|
|||||||
+28
-31
@@ -14,14 +14,20 @@ import sys
|
|||||||
|
|
||||||
from SYS.logger import log, debug
|
from SYS.logger import log, debug
|
||||||
|
|
||||||
from plugins.metadata_provider import (
|
# plugins.metadata_provider is deferred: it transitively loads yt_dlp, Cryptodome,
|
||||||
get_default_subject_scrape_plugin,
|
# imdbinfo, musicbrainzngs and ~1400 modules (~1.5s). Import lazily on first use.
|
||||||
get_metadata_plugin,
|
_METADATA_PROVIDER_MOD: Optional[Any] = None
|
||||||
get_metadata_plugin_for_url,
|
|
||||||
list_metadata_plugins,
|
|
||||||
scrape_isbn_metadata,
|
def _mp() -> Any:
|
||||||
scrape_openlibrary_metadata,
|
"""Return the (lazily imported) plugins.metadata_provider module."""
|
||||||
)
|
global _METADATA_PROVIDER_MOD
|
||||||
|
if _METADATA_PROVIDER_MOD is None:
|
||||||
|
import plugins.metadata_provider as _m
|
||||||
|
_METADATA_PROVIDER_MOD = _m
|
||||||
|
return _METADATA_PROVIDER_MOD
|
||||||
|
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||||
|
|
||||||
@@ -41,11 +47,6 @@ CmdletArg = sh.CmdletArg
|
|||||||
SharedArgs = sh.SharedArgs
|
SharedArgs = sh.SharedArgs
|
||||||
parse_cmdlet_args = sh.parse_cmdlet_args
|
parse_cmdlet_args = sh.parse_cmdlet_args
|
||||||
|
|
||||||
try:
|
|
||||||
from SYS.metadata import extract_title
|
|
||||||
except ImportError:
|
|
||||||
extract_title = None
|
|
||||||
|
|
||||||
|
|
||||||
def _dedup_tags_preserve_order(tags: List[str]) -> List[str]:
|
def _dedup_tags_preserve_order(tags: List[str]) -> List[str]:
|
||||||
"""Deduplicate tags case-insensitively while preserving order."""
|
"""Deduplicate tags case-insensitively while preserving order."""
|
||||||
@@ -210,7 +211,7 @@ def _extract_tag_value(tags_list: List[str], namespace: str) -> Optional[str]:
|
|||||||
|
|
||||||
def _scrape_openlibrary_metadata(olid: str) -> List[str]:
|
def _scrape_openlibrary_metadata(olid: str) -> List[str]:
|
||||||
try:
|
try:
|
||||||
return list(scrape_openlibrary_metadata(olid))
|
return list(_mp().scrape_openlibrary_metadata(olid))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"OpenLibrary scraping error: {e}", file=sys.stderr)
|
log(f"OpenLibrary scraping error: {e}", file=sys.stderr)
|
||||||
return []
|
return []
|
||||||
@@ -218,7 +219,7 @@ def _scrape_openlibrary_metadata(olid: str) -> List[str]:
|
|||||||
|
|
||||||
def _scrape_isbn_metadata(isbn: str) -> List[str]:
|
def _scrape_isbn_metadata(isbn: str) -> List[str]:
|
||||||
try:
|
try:
|
||||||
return list(scrape_isbn_metadata(isbn))
|
return list(_mp().scrape_isbn_metadata(isbn))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"ISBN scraping error: {e}", file=sys.stderr)
|
log(f"ISBN scraping error: {e}", file=sys.stderr)
|
||||||
return []
|
return []
|
||||||
@@ -400,7 +401,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
|
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
|
||||||
plugin = None
|
plugin = None
|
||||||
if scrape_target.startswith(("http://", "https://")):
|
if scrape_target.startswith(("http://", "https://")):
|
||||||
plugin = get_metadata_plugin_for_url(scrape_target, config)
|
plugin = _mp().get_metadata_plugin_for_url(scrape_target, config)
|
||||||
if plugin is None:
|
if plugin is None:
|
||||||
log("No metadata plugin can scrape this URL", file=sys.stderr)
|
log("No metadata plugin can scrape this URL", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
@@ -412,9 +413,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
if scrape_target:
|
if scrape_target:
|
||||||
plugin = get_metadata_plugin(scrape_target, config)
|
plugin = _mp().get_metadata_plugin(scrape_target, config)
|
||||||
else:
|
else:
|
||||||
plugin = get_default_subject_scrape_plugin(config)
|
plugin = _mp().get_default_subject_scrape_plugin(config)
|
||||||
if plugin is None:
|
if plugin is None:
|
||||||
if scrape_target:
|
if scrape_target:
|
||||||
log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
|
log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
|
||||||
@@ -749,7 +750,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
plugin_for_apply = get_metadata_plugin(str(result_provider), config)
|
plugin_for_apply = _mp().get_metadata_plugin(str(result_provider), config)
|
||||||
if plugin_for_apply is not None:
|
if plugin_for_apply is not None:
|
||||||
apply_tags = plugin_for_apply.filter_tags_for_store_apply(
|
apply_tags = plugin_for_apply.filter_tags_for_store_apply(
|
||||||
[str(t) for t in result_tags if t is not None]
|
[str(t) for t in result_tags if t is not None]
|
||||||
@@ -944,18 +945,14 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
_SCRAPE_CHOICES = []
|
_SCRAPE_CHOICES = [
|
||||||
try:
|
"itunes",
|
||||||
_SCRAPE_CHOICES = sorted(list_metadata_plugins().keys())
|
"openlibrary",
|
||||||
except Exception:
|
"googlebooks",
|
||||||
_SCRAPE_CHOICES = [
|
"google",
|
||||||
"itunes",
|
"musicbrainz",
|
||||||
"openlibrary",
|
"imdb",
|
||||||
"googlebooks",
|
]
|
||||||
"google",
|
|
||||||
"musicbrainz",
|
|
||||||
"imdb",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class Get_Tag(Cmdlet):
|
class Get_Tag(Cmdlet):
|
||||||
|
|||||||
+33
-33
@@ -40,49 +40,49 @@ except ImportError:
|
|||||||
PdfWriter = None
|
PdfWriter = None
|
||||||
PdfReader = None
|
PdfReader = None
|
||||||
|
|
||||||
try:
|
# Stub fallbacks used before SYS.metadata is lazily imported (or if unavailable).
|
||||||
from SYS.metadata import (
|
HAS_METADATA_API: bool = False
|
||||||
read_tags_from_file,
|
_metadata_loaded: bool = False
|
||||||
merge_multiple_tag_lists,
|
|
||||||
)
|
|
||||||
|
|
||||||
HAS_METADATA_API = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_METADATA_API = False
|
|
||||||
|
|
||||||
def read_tags_from_file(file_path: Path) -> List[str]:
|
def read_tags_from_file(file_path: Path) -> List[str]:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def write_tags_to_file(
|
|
||||||
file_path: Path,
|
|
||||||
tags: List[str],
|
|
||||||
source_hashes: Optional[List[str]] = None,
|
|
||||||
url: Optional[List[str]] = None,
|
|
||||||
append: bool = False,
|
|
||||||
) -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def dedup_tags_by_namespace(tags: List[str]) -> List[str]:
|
def merge_multiple_tag_lists(sources: List[List[str]],
|
||||||
return tags
|
strategy: str = "first") -> List[str]:
|
||||||
|
out: List[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for src in sources:
|
||||||
|
for t in src or []:
|
||||||
|
s = str(t)
|
||||||
|
if s and s not in seen:
|
||||||
|
out.append(s)
|
||||||
|
seen.add(s)
|
||||||
|
return out
|
||||||
|
|
||||||
def merge_multiple_tag_lists(sources: List[List[str]],
|
|
||||||
strategy: str = "first") -> List[str]:
|
|
||||||
out: List[str] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for src in sources:
|
|
||||||
for t in src or []:
|
|
||||||
s = str(t)
|
|
||||||
if s and s not in seen:
|
|
||||||
out.append(s)
|
|
||||||
seen.add(s)
|
|
||||||
return out
|
|
||||||
|
|
||||||
def write_metadata(*_args: Any, **_kwargs: Any) -> None:
|
def _ensure_metadata_imports() -> None:
|
||||||
return None
|
"""Lazily import SYS.metadata to avoid loading Cryptodome (~1s) at startup."""
|
||||||
|
global _metadata_loaded, HAS_METADATA_API, read_tags_from_file, merge_multiple_tag_lists
|
||||||
|
if _metadata_loaded:
|
||||||
|
return
|
||||||
|
_metadata_loaded = True
|
||||||
|
try:
|
||||||
|
from SYS.metadata import ( # type: ignore[assignment]
|
||||||
|
read_tags_from_file as _rtf,
|
||||||
|
merge_multiple_tag_lists as _mml,
|
||||||
|
)
|
||||||
|
read_tags_from_file = _rtf # type: ignore[assignment]
|
||||||
|
merge_multiple_tag_lists = _mml # type: ignore[assignment]
|
||||||
|
HAS_METADATA_API = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_METADATA_API = False
|
||||||
|
|
||||||
|
|
||||||
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||||
"""Merge multiple files into one."""
|
"""Merge multiple files into one."""
|
||||||
|
_ensure_metadata_imports()
|
||||||
|
|
||||||
# Parse help
|
# Parse help
|
||||||
if should_show_help(args):
|
if should_show_help(args):
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from SYS.result_table import Table
|
from SYS.result_table import Table
|
||||||
|
|
||||||
|
|
||||||
@@ -68,9 +66,10 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
|
|||||||
code = int(getattr(response, "status_code", 0) or 0)
|
code = int(getattr(response, "status_code", 0) or 0)
|
||||||
ok = 200 <= code < 500
|
ok = 200 <= code < 500
|
||||||
return ok, f"{url} (HTTP {code})"
|
return ok, f"{url} (HTTP {code})"
|
||||||
except httpx.TimeoutException:
|
|
||||||
return False, f"{url} (timeout)"
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
import httpx as _httpx
|
||||||
|
if isinstance(exc, _httpx.TimeoutException):
|
||||||
|
return False, f"{url} (timeout)"
|
||||||
return False, f"{url} ({type(exc).__name__})"
|
return False, f"{url} ({type(exc).__name__})"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -875,3 +875,7 @@ try:
|
|||||||
except Exception:
|
except Exception:
|
||||||
# best-effort registration
|
# best-effort registration
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# Backward-compatible alias: tests and callers may import `plugins.matrix.cmdnat`.
|
||||||
|
from . import commands as cmdnat # noqa: E402
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
$tests = Get-ChildItem "$PSScriptRoot\..\tests\test_*.py" | Select-Object -ExpandProperty Name
|
||||||
|
foreach ($f in $tests) {
|
||||||
|
Write-Host "=== $f ===" -NoNewline
|
||||||
|
$out = & c:/Forgejo/Medios-Macina/.venv/Scripts/python.exe -m pytest "tests/$f" -q --tb=line 2>&1 | Select-Object -Last 2
|
||||||
|
Write-Host " $out"
|
||||||
|
}
|
||||||
+32
-9
@@ -4,16 +4,39 @@ This package contains wrappers around external tools (e.g. yt-dlp) so cmdlets ca
|
|||||||
common defaults (cookies, timeouts, format selectors) and users can override them via
|
common defaults (cookies, timeouts, format selectors) and users can override them via
|
||||||
`config.conf`.
|
`config.conf`.
|
||||||
"""
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from .ytdlp import YtDlpTool, YtDlpDefaults
|
# Lazy-loaded to avoid pulling in yt_dlp, playwright, and their heavy transitive
|
||||||
from .playwright import PlaywrightTool, PlaywrightDefaults
|
# dependencies (~1–2 s) at package import time. Each submodule is loaded only when
|
||||||
from .florencevision import FlorenceVisionTool, FlorenceVisionDefaults
|
# a name from it is first accessed through this package namespace.
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"YtDlpTool",
|
"YtDlpTool",
|
||||||
"YtDlpDefaults",
|
"YtDlpDefaults",
|
||||||
"PlaywrightTool",
|
"PlaywrightTool",
|
||||||
"PlaywrightDefaults",
|
"PlaywrightDefaults",
|
||||||
"FlorenceVisionTool",
|
"FlorenceVisionTool",
|
||||||
"FlorenceVisionDefaults",
|
"FlorenceVisionDefaults",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
_MODULE_ATTRS = {
|
||||||
|
"YtDlpTool": ".ytdlp",
|
||||||
|
"YtDlpDefaults": ".ytdlp",
|
||||||
|
"PlaywrightTool": ".playwright",
|
||||||
|
"PlaywrightDefaults": ".playwright",
|
||||||
|
"FlorenceVisionTool": ".florencevision",
|
||||||
|
"FlorenceVisionDefaults": ".florencevision",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> object:
|
||||||
|
submod = _MODULE_ATTRS.get(name)
|
||||||
|
if submod is None:
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
from importlib import import_module
|
||||||
|
mod = import_module(submod, package=__name__)
|
||||||
|
obj = getattr(mod, name)
|
||||||
|
# Cache on this module so subsequent accesses bypass __getattr__.
|
||||||
|
globals()[name] = obj
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user