diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index e4862a8..8b0c686 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -37,7 +37,7 @@ "(rapidgator\\.net/file/[0-9]{7,8})" ], "regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))", - "status": true + "status": false }, "turbobit": { "name": "turbobit", @@ -425,7 +425,7 @@ "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})" ], "regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})", - "status": false + "status": true }, "hot4share": { "name": "hot4share", @@ -482,7 +482,7 @@ "(katfile\\.com/[0-9a-zA-Z]{12})" ], "regexp": "(katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", - "status": false + "status": true }, "mediafire": { "name": "mediafire", @@ -595,7 +595,7 @@ "(simfileshare\\.net/download/[0-9]+/)" ], "regexp": "(simfileshare\\.net/download/[0-9]+/)", - "status": false + "status": true }, "streamtape": { "name": "streamtape", @@ -690,7 +690,7 @@ "uploadrar\\.(net|com)/([0-9a-z]{12})" ], "regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))", - "status": false, + "status": true, "hardRedirect": [ "uploadrar.com/([0-9a-zA-Z]{12})" ] diff --git a/CLI.py b/CLI.py index 5102cf5..05982a7 100644 --- a/CLI.py +++ b/CLI.py @@ -59,13 +59,9 @@ from SYS.rich_display import ( ) from cmdnat._status_shared import ( add_startup_check as _shared_add_startup_check, - default_provider_ping_targets as _default_provider_ping_targets, - has_provider as _has_provider, + collect_plugin_startup_checks as _collect_plugin_startup_checks, has_store_subtype as _has_store_subtype, has_tool as _has_tool, - ping_first as _ping_first, - ping_url as _ping_url, - provider_display_name as _provider_display_name, ) @@ -98,12 +94,12 @@ from SYS.cmdlet_catalog import ( list_cmdlet_metadata, list_cmdlet_names, ) -from SYS.config import load_config +from SYS.config import load_config, resolve_cookies_path from SYS.result_table import Table from SYS.worker import WorkerManagerRegistry, WorkerStages, WorkerOutputMirror, WorkerStageSession from SYS.pipeline import PipelineExecutor -from ProviderCore.registry import provider_inline_query_choices +from ProviderCore.registry import plugin_inline_query_choices @@ -507,25 +503,25 @@ class CmdletIntrospection: if backends: return backends - if normalized_arg == "provider": + if normalized_arg == "plugin": canonical_cmd = (cmd_name or "").replace("_", "-").lower() try: - from ProviderCore.registry import list_search_providers, list_file_providers + from ProviderCore.registry import list_search_plugins, list_upload_plugins except Exception: - list_search_providers = None # type: ignore - list_file_providers = None # type: ignore + list_search_plugins = None # type: ignore + list_upload_plugins = None # type: ignore provider_choices: List[str] = [] - if canonical_cmd in {"add-file"} and list_file_providers is not None: - providers = list_file_providers(config) or {} + if canonical_cmd in {"add-file"} and list_upload_plugins is not None: + providers = list_upload_plugins(config) or {} available = [ name for name, is_ready in providers.items() if is_ready ] return sorted(available) if available else sorted(providers.keys()) - if list_search_providers is not None: - providers = list_search_providers(config) or {} + if list_search_plugins is not None: + providers = list_search_plugins(config) or {} available = [ name for name, is_ready in providers.items() if is_ready ] @@ -680,7 +676,7 @@ class CmdletCompleter(Completer): provider_name = None if cmd_name == "search-file": - provider_name = self._flag_value(stage_tokens, "-provider", "--provider") + provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin") if ( cmd_name == "search-file" @@ -705,7 +701,7 @@ class CmdletCompleter(Completer): field, partial = inline_token.split(":", 1) field = field.strip().lower() partial_lower = partial.strip().lower() - inline_choices = provider_inline_query_choices(provider_name, field, config) + inline_choices = plugin_inline_query_choices(provider_name, field, config) if inline_choices: filtered = ( [c for c in inline_choices if partial_lower in str(c).lower()] @@ -728,7 +724,7 @@ class CmdletCompleter(Completer): if choices: choice_list = choices normalized_prev = prev_token.lstrip("-").strip().lower() - if normalized_prev == "provider" and current_token: + if normalized_prev in {"plugin", "provider"} and current_token: current_lower = current_token.lower() filtered = [c for c in choices if current_lower in c.lower()] if filtered: @@ -1996,188 +1992,13 @@ Come to love it when others take what you share, as there is no greater joy ) if isinstance(config, dict) else None if isinstance(provider_cfg, dict) and provider_cfg: - from Provider.metadata_provider import list_metadata_providers - from ProviderCore.registry import ( - list_file_providers, - list_providers, - list_search_providers, - ) - - provider_availability = list_providers(config) or {} - search_availability = list_search_providers(config) or {} - file_availability = list_file_providers(config) or {} - meta_availability = list_metadata_providers(config) or {} - - already_checked = {"matrix"} - - for provider_name in provider_cfg.keys(): - prov = str(provider_name or "").strip().lower() - if not prov or prov in already_checked: - continue - display = _provider_display_name(prov) - - if prov == "alldebrid": - try: - from Provider.alldebrid import _get_debrid_api_key - from API.alldebrid import AllDebridClient - - api_key = _get_debrid_api_key(config) - if not api_key: - _add_startup_check( - "DISABLED", - display, - provider=prov, - detail="Not configured" - ) - else: - client = AllDebridClient(api_key) - base_url = str( - getattr(client, - "base_url", - "") or "" - ).strip() - _add_startup_check( - "ENABLED", - display, - provider=prov, - detail=base_url or "Connected", - ) - except Exception as exc: - _add_startup_check( - "DISABLED", - display, - provider=prov, - detail=str(exc) - ) - continue - - is_known = False - ok_val: Optional[bool] = None - if prov in provider_availability: - is_known = True - ok_val = bool(provider_availability.get(prov)) - elif prov in search_availability: - is_known = True - ok_val = bool(search_availability.get(prov)) - elif prov in file_availability: - is_known = True - ok_val = bool(file_availability.get(prov)) - elif prov in meta_availability: - is_known = True - ok_val = bool(meta_availability.get(prov)) - - if not is_known: - _add_startup_check( - "UNKNOWN", - display, - provider=prov, - detail="Not registered" - ) - else: - detail = "Configured" if ok_val else "Not configured" - ping_targets = _default_provider_ping_targets(prov) - if ping_targets: - ping_ok, ping_detail = _ping_first(ping_targets) - if ok_val: - detail = ping_detail - else: - detail = ( - (detail + " | " + - ping_detail) if ping_detail else detail - ) - _add_startup_check( - "ENABLED" if ok_val else "DISABLED", - display, - provider=prov, - detail=detail, - ) - - already_checked.add(prov) - - default_search_providers = [ - "openlibrary", - "libgen", - "youtube", - "bandcamp" - ] - for prov in default_search_providers: - if prov in already_checked: - continue - display = _provider_display_name(prov) - ok_val = ( - bool(search_availability.get(prov)) - if prov in search_availability else False - ) - ping_targets = _default_provider_ping_targets(prov) - ping_ok, ping_detail = ( - _ping_first(ping_targets) if ping_targets else (False, "No ping target") - ) - detail = ping_detail or ( - "Available" if ok_val else "Unavailable" - ) - if not ok_val: - detail = "Unavailable" + ( - f" | {ping_detail}" if ping_detail else "" - ) + for check in _collect_plugin_startup_checks(config): _add_startup_check( - "ENABLED" if (ok_val and ping_ok) else "DISABLED", - display, - provider=prov, - detail=detail, - ) - already_checked.add(prov) - - if "0x0" not in already_checked: - ok_val = ( - bool(file_availability.get("0x0")) - if "0x0" in file_availability else False - ) - ping_ok, ping_detail = _ping_url("https://0x0.st") - detail = ping_detail - if not ok_val: - detail = "Unavailable" + ( - f" | {ping_detail}" if ping_detail else "" - ) - _add_startup_check( - "ENABLED" if (ok_val and ping_ok) else "DISABLED", - "0x0", - provider="0x0", - detail=detail, - ) - - if _has_provider(config, "matrix"): - try: - from Provider.matrix import Matrix - - provider = Matrix(config) - matrix_conf = ( - config.get("provider", - {}).get("matrix", - {}) if isinstance(config, - dict) else {} - ) - homeserver = str(matrix_conf.get("homeserver") or "").strip() - room_id = str(matrix_conf.get("room_id") or "").strip() - if homeserver and not homeserver.startswith("http"): - homeserver = f"https://{homeserver}" - target = homeserver.rstrip("/") - if room_id: - target = ( - target + (" " if target else "") - ) + f"room:{room_id}" - _add_startup_check( - "ENABLED" if provider.validate() else "DISABLED", - "Matrix", - provider="matrix", - detail=target or - ("Connected" if provider.validate() else "Not configured"), - ) - except Exception as exc: - _add_startup_check( - "DISABLED", - "Matrix", - provider="matrix", - detail=str(exc) + str(check.get("status") or "UNKNOWN"), + str(check.get("name") or "Plugin"), + provider=str(check.get("plugin") or ""), + detail=str(check.get("detail") or ""), + files=check.get("files"), ) if _has_store_subtype(config, "debrid"): @@ -2213,9 +2034,7 @@ Come to love it when others take what you share, as there is no greater joy ) try: - from tool.ytdlp import YtDlpTool - - cookiefile = YtDlpTool(config).resolve_cookiefile() + cookiefile = resolve_cookies_path(config) if cookiefile is not None: _add_startup_check("FOUND", "Cookies", detail=str(cookiefile)) else: diff --git a/MPV/pipeline_helper.py b/MPV/pipeline_helper.py index 60557c2..fcb63de 100644 --- a/MPV/pipeline_helper.py +++ b/MPV/pipeline_helper.py @@ -67,6 +67,7 @@ from SYS.config import load_config, reload_config # noqa: E402 from SYS.logger import set_debug, debug, set_thread_stream # noqa: E402 from SYS.repl_queue import enqueue_repl_command # noqa: E402 from SYS.utils import format_bytes # noqa: E402 +from ProviderCore.registry import get_plugin, get_plugin_class # noqa: E402 REQUEST_PROP = "user-data/medeia-pipeline-request" RESPONSE_PROP = "user-data/medeia-pipeline-response" @@ -936,39 +937,36 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]: "table": None, } + cfg = load_config() or {} + plugin = get_plugin("ytdlp", cfg) + if plugin is None or not hasattr(plugin, "list_url_formats"): + return { + "success": False, + "stdout": "", + "stderr": "", + "error": "yt-dlp plugin unavailable", + "table": None, + } + try: - from tool.ytdlp import list_formats, is_browseable_format # noqa: WPS433 + formats = plugin.list_url_formats( + url, + no_playlist=True, + timeout_seconds=25, + ) except Exception as exc: return { "success": False, "stdout": "", "stderr": "", - "error": f"yt-dlp tool unavailable: {type(exc).__name__}: {exc}", + "error": f"yt-dlp plugin probe failed: {type(exc).__name__}: {exc}", "table": None, } - cookiefile = None - try: - from tool.ytdlp import YtDlpTool # noqa: WPS433 - - cfg = load_config() or {} - cookie_path = YtDlpTool(cfg).resolve_cookiefile() - if cookie_path is not None: - cookiefile = str(cookie_path) - except Exception: - cookiefile = None - def _format_bytes(n: Any) -> str: """Format bytes using centralized utility.""" return format_bytes(n) - formats = list_formats( - url, - no_playlist=True, - cookiefile=cookiefile, - timeout_seconds=25, - ) - if formats is None: return { "success": False, @@ -990,9 +988,10 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]: }, } - browseable = [f for f in formats if is_browseable_format(f)] - if browseable: - formats = browseable + try: + formats = plugin.filter_picker_formats(formats) + except Exception: + pass # Debug: dump a short summary of the format list to the helper log. try: @@ -2040,8 +2039,19 @@ def main(argv: Optional[list[str]] = None) -> int: # Publish yt-dlp supported domains for Lua menu filtering try: - from tool.ytdlp import _build_supported_domains - domains = sorted(list(_build_supported_domains())) + plugin_class = get_plugin_class("ytdlp") + domains = [] + if plugin_class is not None: + domains = sorted( + { + str(value).strip().lower() + for value in plugin_class.url_patterns() + if isinstance(value, str) + and str(value).strip() + and "://" not in str(value) + and not str(value).strip().endswith(":") + } + ) if domains: # We join them into a space-separated string for Lua to parse easily domains_str = " ".join(domains) diff --git a/Provider/HIFI.py b/Provider/HIFI.py index 1f075a6..103f899 100644 --- a/Provider/HIFI.py +++ b/Provider/HIFI.py @@ -64,8 +64,7 @@ def _format_total_seconds(seconds: Any) -> str: class HIFI(Provider): - - PROVIDER_NAME = "hifi" + PLUGIN_NAME = "hifi" TABLE_AUTO_STAGES = { "hifi.track": ["download-file"], @@ -2091,4 +2090,57 @@ class HIFI(Provider): except Exception: pass - return True \ No newline at end of file + return True + + def expand_selection( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + table_type: str = "", + **_kwargs: Any, + ) -> Optional[List[Any]]: + _ = ctx + if stage_is_last: + return None + + normalized_table = str(table_type or "").strip().lower() + if normalized_table != "hifi.album": + return None + + try: + contexts = self._extract_album_selection_context(selected_items) + except Exception: + return None + if not contexts: + return None + + track_items: List[Any] = [] + seen_track_ids: set[int] = set() + for album_id, album_title, artist_name in contexts: + try: + track_results = self._tracks_for_album( + album_id=album_id, + album_title=album_title, + artist_name=artist_name, + limit=500, + ) + except Exception: + track_results = [] + for track in track_results or []: + try: + metadata = getattr(track, "full_metadata", None) + track_id = None + if isinstance(metadata, dict): + raw_id = metadata.get("trackId") or metadata.get("id") + track_id = int(raw_id) if raw_id is not None else None + if track_id is not None: + if track_id in seen_track_ids: + continue + seen_track_ids.add(track_id) + except Exception: + pass + track_items.append(track) + + return track_items or None \ No newline at end of file diff --git a/Provider/Tidal.py b/Provider/Tidal.py index abb8ca3..5b6c588 100644 --- a/Provider/Tidal.py +++ b/Provider/Tidal.py @@ -64,7 +64,7 @@ def _format_total_seconds(seconds: Any) -> str: class Tidal(Provider): - PROVIDER_NAME = "tidal" + PLUGIN_NAME = "tidal" TABLE_AUTO_STAGES = { "tidal.track": ["download-file"], @@ -82,7 +82,7 @@ class Tidal(Provider): "tidal.com", "listen.tidal.com", ) - URL = URL_DOMAINS + URL = URL_DOMAINS + ("tidal:",) """Provider that targets the Tidal search endpoint. The CLI can supply a list of fail-over URLs via ``provider.tidal.api_urls`` or @@ -210,6 +210,9 @@ class Tidal(Provider): self.api_timeout = 10.0 self.api_clients = [TidalApiClient(base_url=url, timeout=self.api_timeout) for url in self.api_urls] + def resolve_playback_path(self, item: Any, **_kwargs: Any) -> Optional[str]: + return resolve_tidal_manifest_path(item) + def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: """Parse inline `key:value` query arguments. @@ -2398,4 +2401,57 @@ class Tidal(Provider): except Exception: pass - return True \ No newline at end of file + return True + + def expand_selection( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + table_type: str = "", + **_kwargs: Any, + ) -> Optional[List[Any]]: + _ = ctx + if stage_is_last: + return None + + normalized_table = str(table_type or "").strip().lower() + if normalized_table != "tidal.album": + return None + + try: + contexts = self._extract_album_selection_context(selected_items) + except Exception: + return None + if not contexts: + return None + + track_items: List[Any] = [] + seen_track_ids: set[int] = set() + for album_id, album_title, artist_name in contexts: + try: + track_results = self._tracks_for_album( + album_id=album_id, + album_title=album_title, + artist_name=artist_name, + limit=500, + ) + except Exception: + track_results = [] + for track in track_results or []: + try: + metadata = getattr(track, "full_metadata", None) + track_id = None + if isinstance(metadata, dict): + raw_id = metadata.get("trackId") or metadata.get("id") + track_id = int(raw_id) if raw_id is not None else None + if track_id is not None: + if track_id in seen_track_ids: + continue + seen_track_ids.add(track_id) + except Exception: + pass + track_items.append(track) + + return track_items or None \ No newline at end of file diff --git a/Provider/__init__.py b/Provider/__init__.py index bec937c..57b8add 100644 --- a/Provider/__init__.py +++ b/Provider/__init__.py @@ -1,7 +1,7 @@ -"""Provider plugin modules. +"""Built-in plugin modules. -Concrete provider implementations live in this package. -The public entrypoint/registry is ProviderCore.registry. +Concrete built-in plugins live in this package. +The public registry lives in ProviderCore.registry. """ # Register providers with the strict ResultTable adapter system diff --git a/Provider/alldebrid.py b/Provider/alldebrid.py index 83cae0c..1f676c1 100644 --- a/Provider/alldebrid.py +++ b/Provider/alldebrid.py @@ -351,7 +351,7 @@ def _dispatch_alldebrid_magnet_search( if callable(exec_fn): exec_fn( None, - ["-provider", "alldebrid", f"ID={magnet_id}"], + ["-plugin", "alldebrid", f"ID={magnet_id}"], config, ) except Exception: @@ -493,7 +493,7 @@ def download_magnet( def expand_folder_item( item: Any, - get_search_provider: Optional[Callable[[str, Dict[str, Any]], Any]], + get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]], config: Dict[str, Any], ) -> Tuple[List[Any], Optional[str]]: table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table") @@ -517,15 +517,15 @@ def expand_folder_item( except Exception: magnet_id = None - if magnet_id is None or get_search_provider is None: + if magnet_id is None or get_search_plugin is None: return [], None - provider = get_search_provider("alldebrid", config) if get_search_provider else None - if provider is None: + plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None + if plugin is None: return [], None try: - files = provider.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)}) + files = plugin.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)}) except Exception: files = [] @@ -609,7 +609,7 @@ class AllDebrid(TableProviderMixin, Provider): - Drill-down: Selecting a folder row (@N) fetches and displays all files SELECTION FLOW: - 1. User runs: search-file -provider alldebrid "ubuntu" + 1. User runs: search-file -plugin alldebrid "ubuntu" 2. Results show magnet folders and (optionally) files 3. User selects a row: @1 4. Selection metadata routes to download-file with -url alldebrid:magnet: @@ -619,7 +619,7 @@ class AllDebrid(TableProviderMixin, Provider): # Magnet URIs should be routed through this provider. TABLE_AUTO_STAGES = {"alldebrid": ["download-file"]} AUTO_STAGE_USE_SELECTION_ARGS = True - URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲") + URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲", "alldebrid.com") URL_DOMAINS = () def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: @@ -949,12 +949,10 @@ class AllDebrid(TableProviderMixin, Provider): except Exception: return None - @classmethod - def download_for_pipe_result( - cls, + def resolve_pipe_result_download( + self, result: Any, pipe_obj: Optional[PipeObject], - config: Dict[str, Any], ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: """Download a remote provider result on behalf of add-file.""" @@ -1026,8 +1024,7 @@ class AllDebrid(TableProviderMixin, Provider): download_dir = Path(tempfile.mkdtemp(prefix="add-file-alldebrid-")) try: - provider = cls(config) - downloaded_path = provider.download(search_result, download_dir) + downloaded_path = self.download(search_result, download_dir) if not downloaded_path: shutil.rmtree(download_dir, ignore_errors=True) return None, None, None @@ -1049,6 +1046,62 @@ class AllDebrid(TableProviderMixin, Provider): log(f"[alldebrid] add-file download failed: {exc}", file=sys.stderr) shutil.rmtree(download_dir, ignore_errors=True) return None, None, None + + def status_summary(self) -> Dict[str, Any]: + try: + api_key = _get_debrid_api_key(self.config) + if not api_key: + return { + "status": "DISABLED", + "name": self.label, + "plugin": self.name, + "detail": "Not configured", + } + client = AllDebridClient(api_key) + base_url = str(getattr(client, "base_url", "") or "").strip() + return { + "status": "ENABLED", + "name": self.label, + "plugin": self.name, + "detail": base_url or "Connected", + } + except Exception as exc: + return { + "status": "DISABLED", + "name": self.label, + "plugin": self.name, + "detail": str(exc), + } + + def resolve_url(self, url: str, **_kwargs: Any) -> str: + target = str(url or "").strip() + if not target.startswith(("http://", "https://")): + return target + + try: + parsed = urlparse(target) + host = str(parsed.netloc or "").lower() + path = str(parsed.path or "") + except Exception: + return target + + if host != "alldebrid.com" or not path.startswith("/f/"): + return target + + api_key = _get_debrid_api_key(self.config) + if not api_key: + return target + + try: + client = AllDebridClient(str(api_key)) + unlocked = client.unlock_link(target) + if isinstance(unlocked, str) and unlocked.strip(): + return unlocked.strip() + except Exception: + pass + + return target + def download_items( self, result: SearchResult, @@ -1413,7 +1466,7 @@ class AllDebrid(TableProviderMixin, Provider): "provider_view": "files", # Selection metadata for table system "_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], - "_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], + "_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], } results.append( @@ -1528,7 +1581,7 @@ class AllDebrid(TableProviderMixin, Provider): "magnet_name": magnet_name, # Selection metadata: allow @N expansion to drive downloads directly "_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], - "_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], + "_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"], }, ) ) @@ -1629,7 +1682,7 @@ class AllDebrid(TableProviderMixin, Provider): table.set_table_metadata({"provider": "alldebrid", "view": "files", "magnet_id": magnet_id}) except Exception: pass - table.set_source_command("download-file", ["-provider", "alldebrid"]) + table.set_source_command("download-file", ["-plugin", "alldebrid"]) results_payload: List[Dict[str, Any]] = [] for r in files or []: @@ -1662,7 +1715,7 @@ class AllDebrid(TableProviderMixin, Provider): try: - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column def _as_payload(item: Any) -> Dict[str, Any]: @@ -1853,7 +1906,7 @@ try: return ["-title", row.title or ""] - register_provider( + register_plugin( "alldebrid", _adapter, columns=_columns_factory, diff --git a/Provider/example_provider.py b/Provider/example_provider.py index 71f314d..d3bc824 100644 --- a/Provider/example_provider.py +++ b/Provider/example_provider.py @@ -1,4 +1,4 @@ -"""Example provider that uses the new `ResultTable` API. +"""Example plugin that uses the new `ResultTable` API. This module demonstrates a minimal provider adapter that yields `ResultModel` instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer @@ -8,7 +8,7 @@ Run this to see sample output: python -m Provider.example_provider Example usage (piped selector): - provider-table -provider example -sample | select -select 1 | add-file -store default + plugin-table -plugin example -sample | select -select 1 | add-file -store default """ from __future__ import annotations @@ -105,9 +105,9 @@ def selection_fn(row: ResultModel) -> List[str]: return ["-title", row.title] -# Register the provider with the registry so callers can discover it by name -from SYS.result_table_adapters import register_provider -register_provider( +# Register the plugin with the registry so callers can discover it by name +from SYS.result_table_adapters import register_plugin +register_plugin( "example", adapter, columns=columns_factory, @@ -223,17 +223,17 @@ def demo() -> None: def demo_with_selection(idx: int = 0) -> None: - """Demonstrate how a cmdlet would use provider registration and selection args. + """Demonstrate how a cmdlet would use plugin registration and selection args. - - Fetch the registered provider by name + - Fetch the registered plugin by name - Build rows via adapter - Render the table - Show the selection args for the chosen row; these are the args a cmdlet would append when the user picks that row. """ - from SYS.result_table_adapters import get_provider + from SYS.result_table_adapters import get_plugin - provider = get_provider("example") + provider = get_plugin("example") rows = list(provider.adapter(SAMPLE_ITEMS)) cols = provider.get_columns(rows) diff --git a/Provider/fileio.py b/Provider/fileio.py index 0477b04..45efcc5 100644 --- a/Provider/fileio.py +++ b/Provider/fileio.py @@ -50,7 +50,7 @@ def _extract_key(payload: Any) -> Optional[str]: class FileIO(Provider): """File provider for file.io.""" - PROVIDER_NAME = "file.io" + PLUGIN_NAME = "file.io" @classmethod def config_schema(cls) -> List[Dict[str, Any]]: diff --git a/Provider/hello_provider.py b/Provider/hello_provider.py index b005e74..1ec90c3 100644 --- a/Provider/hello_provider.py +++ b/Provider/hello_provider.py @@ -1,6 +1,6 @@ -"""Example provider template for use as a starter kit. +"""Example plugin template for use as a starter kit. -This minimal provider demonstrates the typical hooks a provider may implement: +This minimal plugin demonstrates the typical hooks a plugin may implement: - `validate()` to assert it's usable - `search()` to return `SearchResult` items - `download()` to persist a sample file (useful for local tests) @@ -17,13 +17,14 @@ from ProviderCore.base import Provider, SearchResult class HelloProvider(Provider): - """Very small example provider suitable as a template. + """Very small example plugin suitable as a template. - Table name: `hello` - - Usage: `search-file -provider hello "query"` + - Usage: `search-file -plugin hello "query"` - Selecting a row and piping into `download-file` will call `download()`. """ + PLUGIN_NAME = "hello" URL = ("hello:",) URL_DOMAINS = () diff --git a/Provider/internetarchive.py b/Provider/internetarchive.py index 26f7023..7d2ef66 100644 --- a/Provider/internetarchive.py +++ b/Provider/internetarchive.py @@ -594,9 +594,9 @@ class InternetArchive(Provider): """Internet Archive provider using the `internetarchive` Python module. Supports: - - search-file -provider internetarchive + - search-file -plugin internetarchive - download-file / provider.download() from search results - - add-file -provider internetarchive (uploads) + - add-file -plugin internetarchive (uploads) """ URL = ("archive.org",) diff --git a/Provider/matrix.py b/Provider/matrix.py index dd877d8..d4de26f 100644 --- a/Provider/matrix.py +++ b/Provider/matrix.py @@ -294,7 +294,7 @@ class Matrix(TableProviderMixin, Provider): - MIME detection: Automatic content type classification for Matrix msgtype SELECTION FLOW: - 1. User runs: search-file -provider matrix "room" (or .matrix -list-rooms) + 1. User runs: search-file -plugin matrix "room" (or .matrix -list-rooms) 2. Results show available joined rooms 3. User selects rooms: @1 @2 (or @1,2) 4. Selection triggers upload of pending files to selected rooms @@ -368,6 +368,21 @@ class Matrix(TableProviderMixin, Provider): and matrix_conf.get("access_token") ) + def status_summary(self) -> Dict[str, Any]: + matrix_conf = self.config.get("provider", {}).get("matrix", {}) if isinstance(self.config, dict) else {} + homeserver = str(matrix_conf.get("homeserver") or "").strip() + room_id = str(matrix_conf.get("room_id") or "").strip() + detail = homeserver + if room_id: + detail = (detail + (" " if detail else "")) + f"room:{room_id}" + enabled = bool(self.validate()) + return { + "status": "ENABLED" if enabled else "DISABLED", + "name": self.label, + "plugin": self.name, + "detail": detail or ("Connected" if enabled else "Not configured"), + } + def search( self, query: str, @@ -767,7 +782,7 @@ class Matrix(TableProviderMixin, Provider): # Minimal provider registration for the new table system try: - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column def _convert_search_result_to_model(sr: Any) -> ResultModel: @@ -850,7 +865,7 @@ try: return ["-title", row.title or ""] - register_provider( + register_plugin( "matrix", _adapter, columns=_columns_factory, diff --git a/Provider/metadata_provider.py b/Provider/metadata_provider.py index 8ff46ca..8ba30a4 100644 --- a/Provider/metadata_provider.py +++ b/Provider/metadata_provider.py @@ -40,6 +40,42 @@ except ImportError: # pragma: no cover - optional yt_dlp = None +def _dedup_text_values(values: List[str]) -> List[str]: + out: List[str] = [] + seen: set[str] = set() + for value in values or []: + if value is None: + continue + text = str(value).strip() + if not text: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + out.append(text) + return out + + +def _filter_default_scraped_tags(tags: List[str]) -> List[str]: + blocked = {"title", "artist", "source"} + out: List[str] = [] + seen: set[str] = set() + for tag in tags or []: + text = str(tag or "").strip() + if not text: + continue + namespace = text.split(":", 1)[0].strip().lower() if ":" in text else "" + if namespace in blocked: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + out.append(text) + return out + + class MetadataProvider(ABC): """Base class for metadata providers (music, movies, books, etc.).""" @@ -122,6 +158,64 @@ class MetadataProvider(ABC): return False + def default_subject_scrape_priority(self) -> int: + """Priority used when `get-tag -scrape` is invoked without an explicit provider.""" + + return 0 + + def url_scrape_priority(self, url: str) -> int: + """Priority for handling a raw URL passed to `get-tag -scrape `.""" + + _ = url + return 0 + + def resolve_subject_query( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> Optional[str]: + """Resolve a provider-specific query from the current subject/result.""" + + _ = backend + _ = file_hash + return self.extract_url_query(result, get_field) + + def prefers_store_tag_overwrite(self) -> bool: + """Whether direct subject scrapes should replace the store tag set.""" + + return False + + def filter_tags_for_selection(self, tags: List[str]) -> List[str]: + """Filter scraped tags before presenting a selectable metadata row.""" + + return _filter_default_scraped_tags(tags) + + def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: + """Filter scraped tags before applying them to an existing store-backed item.""" + + return self.filter_tags_for_selection(tags) + + def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: + """Return a URL scrape payload for `get-tag -scrape ` when supported.""" + + items = self.search(url, limit=1) + if not items: + return None + item = items[0] if isinstance(items[0], dict) else {} + try: + tags = [str(t) for t in self.to_tags(item) if t is not None] + except Exception: + tags = [] + return { + "title": item.get("title"), + "tag": _dedup_text_values(tags), + "formats": [], + "playlist_items": [], + } + class ITunesProvider(MetadataProvider): """Metadata provider using the iTunes Search API.""" @@ -1015,6 +1109,226 @@ class YtdlpMetadataProvider(MetadataProvider): def emits_direct_tags(self) -> bool: return True + def default_subject_scrape_priority(self) -> int: + return 100 + + def url_scrape_priority(self, url: str) -> int: + text = str(url or "").strip() + if not text.startswith(("http://", "https://")): + return 0 + return 100 + + def prefers_store_tag_overwrite(self) -> bool: + return True + + def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: + return _dedup_text_values(tags) + + def _resolve_candidate_urls_for_subject( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> List[str]: + try: + from SYS.metadata import normalize_urls + except Exception: + normalize_urls = None # type: ignore[assignment] + + urls: List[str] = [] + + if backend is not None and file_hash: + try: + backend_urls = backend.get_url(file_hash, config=self.config) + if backend_urls: + if normalize_urls: + urls.extend(normalize_urls(backend_urls)) + else: + urls.extend( + [str(u).strip() for u in backend_urls if isinstance(u, str) and str(u).strip()] + ) + except Exception: + pass + + try: + meta = backend.get_metadata(file_hash, config=self.config) + if isinstance(meta, dict) and meta.get("url"): + raw = meta.get("url") + if normalize_urls: + urls.extend(normalize_urls(raw)) + elif isinstance(raw, list): + urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) + elif isinstance(raw, str) and raw.strip(): + urls.append(raw.strip()) + except Exception: + pass + + for key in ("url", "webpage_url", "source_url", "target"): + val = get_field(result, key, None) + if not val: + continue + if normalize_urls: + urls.extend(normalize_urls(val)) + continue + if isinstance(val, str) and val.strip(): + urls.append(val.strip()) + elif isinstance(val, list): + urls.extend([str(u).strip() for u in val if isinstance(u, str) and str(u).strip()]) + + meta_field = get_field(result, "metadata", None) + if isinstance(meta_field, dict) and meta_field.get("url"): + raw = meta_field.get("url") + if normalize_urls: + urls.extend(normalize_urls(raw)) + elif isinstance(raw, list): + urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) + elif isinstance(raw, str) and raw.strip(): + urls.append(raw.strip()) + + return _dedup_text_values(urls) + + def _pick_supported_subject_url(self, urls: List[str]) -> Optional[str]: + if not urls: + return None + + def _is_hydrus_file_url(u: str) -> bool: + text = str(u or "").strip().lower() + return bool(text and "/get_files/file" in text and "hash=" in text) + + candidates = [] + for url in urls: + text = str(url or "").strip() + if not text.startswith(("http://", "https://")): + continue + if _is_hydrus_file_url(text): + continue + candidates.append(text) + if not candidates: + return None + + try: + from tool.ytdlp import is_url_supported_by_ytdlp + + for text in candidates: + try: + if is_url_supported_by_ytdlp(text): + return text + except Exception: + continue + except Exception: + pass + + return candidates[0] if candidates else None + + def resolve_subject_query( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> Optional[str]: + candidate_urls = self._resolve_candidate_urls_for_subject( + result, + get_field, + backend=backend, + file_hash=file_hash, + ) + return self._pick_supported_subject_url(candidate_urls) + + @staticmethod + def _extract_url_formats(formats: Any) -> List[tuple[str, str]]: + if not isinstance(formats, list): + return [] + + video_formats: Dict[str, Dict[str, Any]] = {} + audio_formats: Dict[str, Dict[str, Any]] = {} + + for fmt in formats: + if not isinstance(fmt, dict): + continue + vcodec = fmt.get("vcodec", "none") + acodec = fmt.get("acodec", "none") + height = fmt.get("height") + ext = fmt.get("ext", "unknown") + format_id = fmt.get("format_id", "") + tbr = fmt.get("tbr", 0) + abr = fmt.get("abr", 0) + + if vcodec and vcodec != "none" and height: + if int(height) < 480: + continue + res_key = f"{int(height)}p" + if res_key not in video_formats or tbr > video_formats[res_key].get("tbr", 0): + video_formats[res_key] = { + "label": f"{int(height)}p ({ext})", + "format_id": str(format_id), + "tbr": tbr, + } + elif acodec and acodec != "none" and (not vcodec or vcodec == "none"): + audio_key = f"audio_{abr}" + if audio_key not in audio_formats or abr > audio_formats[audio_key].get("abr", 0): + audio_formats[audio_key] = { + "label": f"audio ({ext})", + "format_id": str(format_id), + "abr": abr, + } + + result: List[tuple[str, str]] = [] + for res in sorted(video_formats.keys(), key=lambda value: int(value.replace("p", "")), reverse=True): + fmt = video_formats[res] + result.append((str(fmt.get("label") or res), str(fmt.get("format_id") or ""))) + if audio_formats: + best_audio_key = max(audio_formats.keys(), key=lambda key: float(audio_formats[key].get("abr", 0) or 0)) + fmt = audio_formats[best_audio_key] + result.append((str(fmt.get("label") or "audio"), str(fmt.get("format_id") or ""))) + return [entry for entry in result if entry[1]] + + @staticmethod + def _build_playlist_items(raw: Dict[str, Any]) -> List[Dict[str, Any]]: + entries = raw.get("entries") + if not isinstance(entries, list): + return [] + + playlist_items: List[Dict[str, Any]] = [] + for idx, entry in enumerate(entries, 1): + if not isinstance(entry, dict): + continue + playlist_items.append( + { + "index": idx, + "id": entry.get("id", f"track_{idx}"), + "title": entry.get("title", entry.get("id", f"Track {idx}")), + "duration": entry.get("duration", 0), + "url": entry.get("url") or entry.get("webpage_url", ""), + } + ) + return playlist_items + + def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: + info = self._extract_info(url) + if not isinstance(info, dict): + return None + + item = { + "title": info.get("title") or "", + "artist": str(info.get("artist") or info.get("uploader") or info.get("channel") or ""), + "album": str(info.get("album") or info.get("playlist_title") or ""), + "year": str((str(info.get("release_date") or "") or str(info.get("upload_date") or ""))[:4]), + "provider": self.name, + "url": str(url or "").strip(), + "raw": info, + } + tags = _dedup_text_values([str(tag) for tag in self.to_tags(item) if tag is not None]) + return { + "title": item.get("title") or None, + "tag": tags, + "formats": self._extract_url_formats(info.get("formats", [])), + "playlist_items": self._build_playlist_items(info), + } + def _coerce_archive_field_list(value: Any) -> List[str]: """Coerce an Archive.org metadata field to a list of strings.""" @@ -1420,7 +1734,7 @@ try: from typing import Iterable from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin def _ensure_search_result(item: Any) -> SearchResult: if isinstance(item, SearchResult): @@ -1526,7 +1840,7 @@ try: return ["-url", url] return ["-title", row.title or ""] - register_provider( + register_plugin( "openlibrary", _adapter, columns=_columns_factory, @@ -1671,3 +1985,42 @@ def get_metadata_provider(name: str, except Exception as exc: log(f"Provider init failed for '{name}': {exc}", file=sys.stderr) return None + + +def get_default_subject_scrape_provider( + config: Optional[Dict[str, Any]] = None, +) -> Optional[MetadataProvider]: + best_provider: Optional[MetadataProvider] = None + best_priority = 0 + for cls in _METADATA_PROVIDERS.values(): + try: + provider = cls(config) + priority = int(provider.default_subject_scrape_priority()) + except Exception: + continue + if priority > best_priority: + best_priority = priority + best_provider = provider + return best_provider + + +def get_metadata_provider_for_url( + url: str, + config: Optional[Dict[str, Any]] = None, +) -> Optional[MetadataProvider]: + text = str(url or "").strip() + if not text: + return None + + best_provider: Optional[MetadataProvider] = None + best_priority = 0 + for cls in _METADATA_PROVIDERS.values(): + try: + provider = cls(config) + priority = int(provider.url_scrape_priority(text)) + except Exception: + continue + if priority > best_priority: + best_priority = priority + best_provider = provider + return best_provider diff --git a/Provider/soulseek.py b/Provider/soulseek.py index f5dd358..b694632 100644 --- a/Provider/soulseek.py +++ b/Provider/soulseek.py @@ -216,7 +216,7 @@ def _suppress_aioslsk_noise() -> Any: class Soulseek(Provider): TABLE_AUTO_STAGES = { - "soulseek": ["download-file", "-provider", "soulseek"], + "soulseek": ["download-file", "-plugin", "soulseek"], } """Search provider for Soulseek P2P network.""" @@ -623,7 +623,7 @@ class Soulseek(Provider): media_kind="audio", size_bytes=item["size"], columns=columns, - selection_action=["download-file", "-provider", "soulseek"], + selection_action=["download-file", "-plugin", "soulseek"], full_metadata={ "username": item["username"], "filename": item["filename"], diff --git a/Provider/vimm.py b/Provider/vimm.py index a3d5ad1..5f188c0 100644 --- a/Provider/vimm.py +++ b/Provider/vimm.py @@ -37,15 +37,15 @@ class Vimm(TableProviderMixin, Provider): 2) Each row carries explicit selection args: `['-url', '']`. Using an explicit `-url` flag avoids ambiguity during argument parsing (some cmdlets accept positional URLs, others accept flags). - 3) The CLI's expansion logic places selection args *before* provider - source args (e.g., `-provider vimm`) so the first positional token is - the intended URL (not an unknown flag like `-provider`). + 3) The CLI's expansion logic places selection args *before* plugin + source args (e.g., `-plugin vimm`) so the first positional token is + the intended URL (not an unknown flag like `-plugin`). - Why this approach? Argument parsing treats the *first* unrecognized token - as a positional value (commonly interpreted as a URL). If a provider - injects hints like `-provider vimm` *before* a bare URL, the parser can - misinterpret `-provider` as the URL, causing confusing attempts to - download `-provider`. By using `-url` and ensuring the URL appears first + as a positional value (commonly interpreted as a URL). If a plugin + injects hints like `-plugin vimm` *before* a bare URL, the parser can + misinterpret `-plugin` as the URL, causing confusing attempts to + download `-plugin`. By using `-url` and ensuring the URL appears first we avoid that class of bugs and make `@N` -> `download-file`/`add-file` flows reliable. @@ -56,7 +56,7 @@ class Vimm(TableProviderMixin, Provider): URL_DOMAINS = ("vimm.net",) def get_source_command(self, args_list: List[str]) -> Tuple[str, List[str]]: - return "search-file", ["-provider", self.name] + return "search-file", ["-plugin", self.name] REGION_CHOICES = [ {"value": "1", "text": "Argentina"}, @@ -807,7 +807,7 @@ class Vimm(TableProviderMixin, Provider): # Minimal provider registration try: - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin from SYS.result_table_api import ResultModel, title_column, metadata_column def _convert_search_result_to_model(sr): @@ -857,7 +857,7 @@ try: return ["-title", row.title or ""] - register_provider( + register_plugin( "vimm", _adapter, columns=_columns_factory, diff --git a/Provider/youtube.py b/Provider/youtube.py index 06b6c4a..7aaa6a9 100644 --- a/Provider/youtube.py +++ b/Provider/youtube.py @@ -21,7 +21,7 @@ class YouTube(TableProviderMixin, Provider): - _selection_args: For @N expansion control and download-file routing SELECTION FLOW: - 1. User runs: search-file -provider youtube "linux tutorial" + 1. User runs: search-file -plugin youtube "linux tutorial" 2. Results show video rows with uploader, duration, views 3. User selects a video: @1 4. Selection metadata routes to download-file with the YouTube URL @@ -121,7 +121,7 @@ class YouTube(TableProviderMixin, Provider): # Minimal provider registration for the new table system try: - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column def _convert_search_result_to_model(sr: Any) -> ResultModel: @@ -206,7 +206,7 @@ try: return ["-title", row.title or ""] - register_provider( + register_plugin( "youtube", _adapter, columns=_columns_factory, diff --git a/Provider/ytdlp.py b/Provider/ytdlp.py index ff61ad8..177c6c1 100644 --- a/Provider/ytdlp.py +++ b/Provider/ytdlp.py @@ -1,95 +1,525 @@ -"""ytdlp format selector provider. +"""yt-dlp search and download plugin. -When a URL is passed through download-file, this provider displays available formats -in a table and routes format selection back to download-file with the chosen format -already specified via -query "format:", skipping the format table on the second invocation. - -This keeps format selection logic in ytdlp and leaves add-file plug-and-play. +This plugin owns all yt-dlp-specific search, picker, and download behavior so +cmdlets can treat it as a generic URL-handling plugin. """ from __future__ import annotations -from typing import Any, Dict, Iterable, List, Optional, Tuple +import re +import sys +from contextlib import AbstractContextManager, nullcontext +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from urllib.parse import urlparse from ProviderCore.base import Provider, SearchResult from SYS.provider_helpers import TableProviderMixin -from SYS.logger import debug +from SYS.logger import debug, log +from SYS.models import DownloadError, DownloadMediaResult, DownloadOptions +from SYS.payload_builders import build_file_result_payload, build_table_result_payload +from SYS.pipeline_progress import PipelineProgress +from SYS.result_table import Table +from SYS.rich_display import stderr_console as get_stderr_console +from SYS import pipeline as pipeline_context +from SYS.utils import sha256_file +from tool.ytdlp import ( + YtDlpTool, + _best_subtitle_sidecar, + _SUBTITLE_EXTS, + _download_with_timeout, + _format_chapters_note, + _read_text_file, + format_for_table_selection, + is_browseable_format, + is_url_supported_by_ytdlp, + list_formats, + probe_url, +) + +_FORMAT_INDEX_RE = re.compile(r"^\s*#?\d+\s*$") + + +def _parse_keyed_csv_spec(spec: str, *, default_key: str) -> Dict[str, List[str]]: + out: Dict[str, List[str]] = {} + text = str(spec or "").strip() + if not text: + return out + + active = str(default_key or "").strip().lower() or "clip" + key_pattern = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*(.*)$") + + for raw_piece in text.split(","): + piece = raw_piece.strip() + if not piece: + continue + + match = key_pattern.match(piece) + if match: + active = (match.group(1) or "").strip().lower() or active + value = (match.group(2) or "").strip() + if value: + out.setdefault(active, []).append(value) + continue + + out.setdefault(active, []).append(piece) + + return out + + +def _parse_query_keyed_spec(query_spec: Optional[str]) -> Dict[str, List[str]]: + if not query_spec: + return {} + keyed = _parse_keyed_csv_spec(str(query_spec), default_key="hash") + if not keyed: + return {} + + def _alias(src: str, dest: str) -> None: + values = keyed.get(src) + if not values: + return + keyed.setdefault(dest, []).extend(list(values)) + keyed.pop(src, None) + + for src in ("range", "ranges", "section", "sections"): + _alias(src, "clip") + for src in ("fmt", "f"): + _alias(src, "format") + for src in ("aud", "a"): + _alias(src, "audio") + + return keyed + + +def _to_seconds(ts: str) -> Optional[int]: + text = str(ts or "").strip() + if not text: + return None + + unit_match = re.fullmatch( + r"(?i)\s*(?:(?P\d+)h)?\s*(?:(?P\d+)m)?\s*(?:(?P\d+(?:\.\d+)?)s)?\s*", + text, + ) + if unit_match and unit_match.group(0).strip() and any(unit_match.group(g) for g in ("h", "m", "s")): + try: + hours = int(unit_match.group("h") or 0) + minutes = int(unit_match.group("m") or 0) + seconds = float(unit_match.group("s") or 0) + return int((hours * 3600) + (minutes * 60) + seconds) + except Exception: + return None + + if ":" in text: + parts = [p.strip() for p in text.split(":")] + if len(parts) == 2: + hh_s = "0" + mm_s, ss_s = parts + elif len(parts) == 3: + hh_s, mm_s, ss_s = parts + else: + return None + try: + hours = int(hh_s) + minutes = int(mm_s) + seconds = float(ss_s) + return int((hours * 3600) + (minutes * 60) + seconds) + except Exception: + return None + + try: + return int(float(text)) + except Exception: + return None + + +def _parse_time_ranges(spec: str) -> List[tuple[int, int]]: + ranges: List[tuple[int, int]] = [] + if not spec: + return ranges + + for piece in str(spec).split(","): + piece = piece.strip() + if not piece or "-" not in piece: + return [] + start_s, end_s = [p.strip() for p in piece.split("-", 1)] + start = _to_seconds(start_s) + end = _to_seconds(end_s) + if start is None or end is None or start >= end: + return [] + ranges.append((start, end)) + return ranges + + +def _build_clip_sections_spec(clip_ranges: Optional[List[tuple[int, int]]]) -> Optional[str]: + if not clip_ranges: + return None + return ",".join(f"{start_s}-{end_s}" for start_s, end_s in clip_ranges) + + +def _format_timecode(seconds: int, *, force_hours: bool) -> str: + total = max(0, int(seconds)) + minutes, secs = divmod(total, 60) + hours, minutes = divmod(minutes, 60) + if force_hours: + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + return f"{minutes:02d}:{secs:02d}" + + +def _rebase_subtitle_timestamp_text(text: str, offset_seconds: int) -> str: + if not text: + return text + try: + offset_value = float(offset_seconds) + except Exception: + return text + if offset_value <= 0: + return text + + timestamp_re = re.compile(r"(?(?:\d{2}:)?\d{2}:\d{2}(?:[\.,]\d{1,3})?)(?!\d)") + + def _shift(match: re.Match[str]) -> str: + original = str(match.group("ts") or "") + if not original: + return original + + frac_sep = "." + frac_digits = 0 + base = original + frac_seconds = 0.0 + if "." in original: + base, frac = original.split(".", 1) + frac_sep = "." + frac_digits = len(frac) + frac_seconds = float(f"0.{frac}") if frac else 0.0 + elif "," in original: + base, frac = original.split(",", 1) + frac_sep = "," + frac_digits = len(frac) + frac_seconds = float(f"0.{frac}") if frac else 0.0 + + parts = base.split(":") + if len(parts) == 3: + hours_s, minutes_s, seconds_s = parts + include_hours = True + elif len(parts) == 2: + hours_s = "0" + minutes_s, seconds_s = parts + include_hours = False + else: + return original + + total = ( + (int(hours_s) * 3600) + + (int(minutes_s) * 60) + + int(seconds_s) + + frac_seconds + + offset_value + ) + total = max(0.0, total) + whole_seconds = int(total) + fraction = total - whole_seconds + hours, remainder = divmod(whole_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + if frac_digits > 0: + scale = 10 ** frac_digits + frac_value = int(round(fraction * scale)) + if frac_value >= scale: + frac_value = 0 + seconds += 1 + if seconds >= 60: + seconds = 0 + minutes += 1 + if minutes >= 60: + minutes = 0 + hours += 1 + frac_text = f"{frac_value:0{frac_digits}d}" + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + return f"{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + try: + return timestamp_re.sub(_shift, str(text)) + except Exception: + return text + + +def _format_clip_range(start_s: int, end_s: int) -> str: + force_hours = bool(start_s >= 3600 or end_s >= 3600) + return f"{_format_timecode(start_s, force_hours=force_hours)}-{_format_timecode(end_s, force_hours=force_hours)}" + + +def _apply_clip_decorations(pipe_objects: List[Dict[str, Any]], clip_ranges: List[tuple[int, int]]) -> None: + if not pipe_objects or len(pipe_objects) != len(clip_ranges): + return + + for po, (start_s, end_s) in zip(pipe_objects, clip_ranges): + clip_range = _format_clip_range(start_s, end_s) + clip_tag = f"clip:{clip_range}" + po["title"] = clip_tag + + tags = po.get("tag") + if not isinstance(tags, list): + tags = [] + tags = [t for t in tags if not str(t).strip().lower().startswith("title:")] + tags = [t for t in tags if not str(t).strip().lower().startswith("relationship:")] + tags.insert(0, f"title:{clip_tag}") + if clip_tag not in tags: + tags.append(clip_tag) + po["tag"] = tags + + notes = po.get("notes") + if isinstance(notes, dict): + sub_text = notes.get("sub") + if isinstance(sub_text, str) and sub_text.strip(): + notes["sub"] = _rebase_subtitle_timestamp_text(sub_text, start_s) + po["notes"] = notes + + if len(pipe_objects) < 2: + return + + hashes: List[str] = [] + for po in pipe_objects: + try: + hashes.append(str(po.get("hash") or "").strip().lower()) + except Exception: + hashes.append("") + king_hash = hashes[0] if hashes and hashes[0] else None + if not king_hash: + return + alt_hashes = [h for h in hashes if h and h != king_hash] + if not alt_hashes: + return + for po in pipe_objects: + po["relationships"] = {"king": [king_hash], "alt": list(alt_hashes)} + + +def _cookiefile_str(ytdlp_tool: YtDlpTool) -> Optional[str]: + try: + cookie_path = ytdlp_tool.resolve_cookiefile() + if cookie_path is not None and cookie_path.is_file(): + return str(cookie_path) + except Exception: + pass + return None + + +def _list_formats_cached( + url: str, + *, + playlist_items_value: Optional[str], + formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], + ytdlp_tool: YtDlpTool, +) -> Optional[List[Dict[str, Any]]]: + key = f"{url}||{playlist_items_value or ''}" + if key in formats_cache: + return formats_cache[key] + fmts = list_formats( + url, + no_playlist=False, + playlist_items=playlist_items_value, + cookiefile=_cookiefile_str(ytdlp_tool), + ) + formats_cache[key] = fmts + return fmts + + +def _format_id_for_query_index( + query_format: str, + url: str, + formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], + ytdlp_tool: YtDlpTool, +) -> Optional[str]: + if not query_format or not _FORMAT_INDEX_RE.match(str(query_format)): + return None + + s_val = str(query_format).strip() + idx = int(s_val.lstrip("#")) + fmts = _list_formats_cached( + url, + playlist_items_value=None, + formats_cache=formats_cache, + ytdlp_tool=ytdlp_tool, + ) + if not fmts: + raise ValueError("Unable to list formats for the URL") + + if s_val and not s_val.startswith("#"): + if any(str(f.get("format_id", "")) == s_val for f in fmts): + return s_val + + candidate_formats = [f for f in fmts if is_browseable_format(f)] + filtered_formats = candidate_formats if candidate_formats else list(fmts) + if idx <= 0 or idx > len(filtered_formats): + raise ValueError(f"Format index {idx} out of range") + + chosen = filtered_formats[idx - 1] + selection_format_id = str(chosen.get("format_id") or "").strip() + if not selection_format_id: + raise ValueError("Selected format has no format_id") + try: + vcodec = str(chosen.get("vcodec", "none")) + acodec = str(chosen.get("acodec", "none")) + if vcodec != "none" and acodec == "none": + selection_format_id = f"{selection_format_id}+bestaudio" + except Exception: + pass + return selection_format_id + + +def _merge_query_args(selection_args: List[str], query_value: str) -> List[str]: + if not query_value: + return selection_args + merged = list(selection_args or []) + if "-query" in merged: + idx_query = merged.index("-query") + if idx_query + 1 < len(merged): + existing = str(merged[idx_query + 1] or "").strip() + merged[idx_query + 1] = f"{existing},{query_value}" if existing else query_value + else: + merged.append(query_value) + else: + merged.extend(["-query", query_value]) + return merged + + +def _build_pipe_objects( + result_obj: Any, + *, + url: str, + opts: DownloadOptions, + embed_chapters: bool, + write_sub: bool, +) -> List[Dict[str, Any]]: + results_to_emit: List[Any] + if isinstance(result_obj, list): + results_to_emit = list(result_obj) + else: + paths = getattr(result_obj, "paths", None) + if isinstance(paths, list) and paths: + results_to_emit = [] + for p in paths: + try: + p_path = Path(p) + except Exception: + continue + try: + if p_path.suffix.lower() in _SUBTITLE_EXTS: + continue + except Exception: + pass + if not p_path.exists() or p_path.is_dir(): + continue + try: + hv = sha256_file(p_path) + except Exception: + hv = None + results_to_emit.append( + DownloadMediaResult( + path=p_path, + info=getattr(result_obj, "info", {}) or {}, + tag=list(getattr(result_obj, "tag", []) or []), + source_url=getattr(result_obj, "source_url", None) or opts.url, + hash_value=hv, + ) + ) + else: + results_to_emit = [result_obj] + + pipe_objects: List[Dict[str, Any]] = [] + pipe_seq = 0 + for downloaded in results_to_emit: + info: Dict[str, Any] = downloaded.info if isinstance(getattr(downloaded, "info", None), dict) else {} + media_path = Path(downloaded.path) + hash_value = getattr(downloaded, "hash_value", None) or sha256_file(media_path) + title = info.get("title") or media_path.stem + tag = list(getattr(downloaded, "tag", []) or []) + if title and f"title:{title}" not in tag: + tag.insert(0, f"title:{title}") + + final_url = None + try: + page_url = info.get("webpage_url") or info.get("original_url") or info.get("url") + if page_url: + final_url = str(page_url) + except Exception: + final_url = None + if not final_url: + final_url = str(url) + + po = build_file_result_payload( + title=title, + path=str(media_path), + hash_value=hash_value, + url=final_url, + tag=tag, + store=getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH", + action="cmdlet:download-file", + is_temp=True, + ytdl_format=getattr(opts, "ytdl_format", None), + media_kind="video" if opts.mode == "video" else "audio", + ) + pipe_seq += 1 + po.setdefault("pipe_index", pipe_seq) + + if embed_chapters: + chapters_text = _format_chapters_note(info) + if chapters_text: + notes = po.get("notes") + if not isinstance(notes, dict): + notes = {} + notes.setdefault("chapters", chapters_text) + po["notes"] = notes + + if write_sub: + try: + sub_path = _best_subtitle_sidecar(media_path) + except Exception: + sub_path = None + if sub_path is not None: + sub_text = _read_text_file(sub_path) + if sub_text: + notes = po.get("notes") + if not isinstance(notes, dict): + notes = {} + notes["sub"] = sub_text + po["notes"] = notes + try: + sub_path.unlink() + except Exception: + pass + + pipe_objects.append(po) + return pipe_objects class ytdlp(TableProviderMixin, Provider): - """ytdlp format selector and video search provider. - - DUAL FUNCTIONALITY: - 1. FORMAT SELECTION: When download-file is used with a yt-dlp supported URL, - displays available formats in a table for user selection. - 2. SEARCH: When search-file is used with -provider ytdlp, searches YouTube - (and other yt-dlp supported sites) for videos. - - FORMAT SELECTION USAGE: - - User runs: download-file "https://example.com/video" - - If URL is ytdlp-supported and no format specified, displays format table - - User selects @N (e.g., @3 for format index 3) - - Selection args include -query "format:", re-invoking download-file - - Second download-file call sees the format query and skips the table, downloads directly - - SEARCH USAGE: - - User runs: search-file -provider ytdlp "linux tutorial" - - Shows YouTube search results as a table - - User selects @1 to download that video - - Selection args route to download-file for streaming download - - SELECTION FLOW (Format): - 1. download-file receives URL without a format query - 2. Calls ytdlp to list formats - 3. Returns formats as ResultTable (from this provider) - 4. User selects @N - 5. Selection args: ["-query", "format:"] route back to download-file - 6. Second download-file invocation with format query skips table - - SELECTION FLOW (Search): - 1. search-file lists YouTube videos via yt_dlp - 2. Returns videos as ResultTable (from this provider) - 3. User selects @N - 4. Selection args: ["-url", ""] route to download-file - 5. download-file downloads the selected video - - TABLE AUTO-STAGES: - - Format selection: ytdlp.formatlist -> download-file (with -query format:) - - Video search: ytdlp.search -> download-file (with -url) - - SUPPORTED URLS: - This provider dynamically discovers all yt-dlp supported sites via yt_dlp.gen_extractors(). - """ + """yt-dlp-backed search and direct download plugin.""" - # Dynamically load URL domains from yt-dlp's extractors - # This enables provider auto-discovery for format selection routing @classmethod def url_patterns(cls) -> Tuple[str, ...]: - """Return supported domains from yt-dlp extractors.""" try: import yt_dlp - # Build a comprehensive list from known extractors and fallback domains - domains = set(cls._fallback_domains) - # Try to get extractors and extract domain info + domains = set(cls._fallback_domains) try: extractors = yt_dlp.gen_extractors() for extractor_class in extractors: - # Get extractor name and try to convert to domain name = getattr(extractor_class, "IE_NAME", "") if name and name not in ("generic", "http"): - # Convert extractor name to domain (e.g., 'YouTube' -> 'youtube.com') name_lower = name.lower().replace("ie", "").strip() if name_lower and len(name_lower) > 2: domains.add(f"{name_lower}.com") except Exception: pass - return tuple(domains) if domains else tuple(cls._fallback_domains) except Exception: return tuple(cls._fallback_domains) - - # Fallback common domains in case extraction fails + _fallback_domains = [ "youtube.com", "youtu.be", "bandcamp.com", @@ -104,23 +534,8 @@ class ytdlp(TableProviderMixin, Provider): "ytdlp.formatlist": ["download-file"], "ytdlp.search": ["download-file"], } - # Forward selection args (including -query format:... or -url) to the next stage AUTO_STAGE_USE_SELECTION_ARGS = True - def search( - self, - query: str, - limit: int = 50, - filters: Optional[Dict[str, Any]] = None, - **kwargs: Any, - ) -> List[SearchResult]: - """ - NOT USED: This provider is invoked via ResultTable integration, not search. - Formats are fetched directly in download-file and returned as ResultTable rows - with this provider registered as the handler. - """ - return [] - def search( self, query: str, @@ -128,18 +543,15 @@ class ytdlp(TableProviderMixin, Provider): filters: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> List[SearchResult]: - """Search YouTube and other yt-dlp supported sites for videos. - - Uses yt-dlp's ytsearch capability to find videos, then returns them - as SearchResult rows for table display and selection. - """ + _ = filters + _ = kwargs try: import yt_dlp # type: ignore ydl_opts: Dict[str, Any] = { "quiet": True, "skip_download": True, - "extract_flat": True + "extract_flat": True, } with yt_dlp.YoutubeDL(ydl_opts) as ydl: # type: ignore[arg-type] search_query = f"ytsearch{limit}:{query}" @@ -149,17 +561,11 @@ class ytdlp(TableProviderMixin, Provider): for video_data in entries[:limit]: title = video_data.get("title", "Unknown") video_id = video_data.get("id", "") - url = video_data.get( - "url" - ) or f"https://youtube.com/watch?v={video_id}" + url = video_data.get("url") or f"https://youtube.com/watch?v={video_id}" uploader = video_data.get("uploader", "Unknown") duration = video_data.get("duration", 0) view_count = video_data.get("view_count", 0) - - duration_str = ( - f"{int(duration // 60)}:{int(duration % 60):02d}" - if duration else "" - ) + duration_str = f"{int(duration // 60)}:{int(duration % 60):02d}" if duration else "" views_str = f"{view_count:,}" if view_count else "" results.append( @@ -181,7 +587,6 @@ class ytdlp(TableProviderMixin, Provider): "uploader": uploader, "duration": duration, "view_count": view_count, - # Selection metadata for table system and @N expansion "_selection_args": ["-url", url], }, ) @@ -192,58 +597,740 @@ class ytdlp(TableProviderMixin, Provider): return [] def validate(self) -> bool: - """Validate yt-dlp availability.""" + return True + + def list_url_formats(self, url: str, **kwargs: Any) -> Optional[List[Dict[str, Any]]]: + url_str = str(url or "").strip() + if not url_str: + return None + + no_playlist = bool(kwargs.get("no_playlist", True)) + timeout_seconds = kwargs.get("timeout_seconds") + playlist_items = kwargs.get("playlist_items") + + ytdlp_tool = YtDlpTool(self.config) + cookiefile = _cookiefile_str(ytdlp_tool) + + call_kwargs: Dict[str, Any] = { + "no_playlist": no_playlist, + "playlist_items": playlist_items, + "cookiefile": cookiefile, + } + if timeout_seconds is not None: + call_kwargs["timeout_seconds"] = timeout_seconds + try: - return True - except Exception: + formats = list_formats(url_str, **call_kwargs) + except TypeError: + call_kwargs.pop("timeout_seconds", None) + formats = list_formats(url_str, **call_kwargs) + return formats if isinstance(formats, list) else None + + def filter_picker_formats( + self, + formats: List[Dict[str, Any]], + **_kwargs: Any, + ) -> List[Dict[str, Any]]: + if not isinstance(formats, list): + return [] + browseable = [fmt for fmt in formats if isinstance(fmt, dict) and is_browseable_format(fmt)] + return browseable if browseable else list(formats) + + def enrich_playlist_entries( + self, + entries: List[Dict[str, Any]], + **_kwargs: Any, + ) -> Optional[List[Dict[str, Any]]]: + if not entries: + return [] + + enriched: List[Dict[str, Any]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + + entry_url = entry.get("url") + if not isinstance(entry_url, str) or not entry_url.strip(): + enriched.append(entry) + continue + + try: + import yt_dlp + + ydl_opts: Dict[str, Any] = { + "quiet": True, + "no_warnings": True, + "skip_download": True, + "noprogress": True, + "socket_timeout": 5, + "retries": 1, + } + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + full_info = ydl.extract_info(entry_url, download=False) + if isinstance(full_info, dict): + enriched.append(full_info) + continue + except Exception: + debug(f"[ytdlp] failed to fetch full metadata for entry URL: {entry_url}") + + enriched.append(entry) + + return enriched + + def _show_playlist_table(self, *, url: str, ytdlp_tool: YtDlpTool) -> bool: + ctx = pipeline_context.get_stage_context() + if ctx is not None and getattr(ctx, "total_stages", 0) > 1: return False + try: + pr = probe_url(url, no_playlist=False, timeout_seconds=15, cookiefile=_cookiefile_str(ytdlp_tool)) + except Exception: + pr = None + if not isinstance(pr, dict): + return False + + entries = pr.get("entries") + if not isinstance(entries, list) or len(entries) <= 1: + return False + + extractor_name = str(pr.get("extractor") or pr.get("extractor_key") or "").strip().lower() + table_type: Optional[str] = None + if "bandcamp" in extractor_name: + table_type = "bandcamp" + elif "youtube" in extractor_name: + table_type = "youtube" + + def _entry_to_url(entry: Any) -> Optional[str]: + if not isinstance(entry, dict): + return None + for key in ("webpage_url", "original_url", "url"): + value = entry.get(key) + if isinstance(value, str) and value.strip(): + cleaned = value.strip() + try: + if urlparse(cleaned).scheme in {"http", "https"}: + return cleaned + except Exception: + return cleaned + entry_id = entry.get("id") + if isinstance(entry_id, str) and entry_id.strip() and "youtube" in extractor_name: + return f"https://www.youtube.com/watch?v={entry_id.strip()}" + return None + + table = Table(preserve_order=True) + safe_url = str(url or "").strip() + table.title = f'download-file -url "{safe_url}"' if safe_url else "download-file" + if table_type: + try: + table.set_table(table_type) + except Exception: + table.table = table_type + table.set_source_command("download-file", []) + try: + table._perseverance(True) + except Exception: + pass + + results_list: List[Dict[str, Any]] = [] + for idx, entry in enumerate(entries[:200], 1): + title = entry.get("title") if isinstance(entry, dict) else None + uploader = entry.get("uploader") if isinstance(entry, dict) else None + duration = entry.get("duration") if isinstance(entry, dict) else None + entry_url = _entry_to_url(entry) + row = build_table_result_payload( + table="download-file", + title=str(title or f"Item {idx}"), + detail=str(uploader or ""), + columns=[ + ("#", str(idx)), + ("Title", str(title or "")), + ("Duration", str(duration or "")), + ("Uploader", str(uploader or "")), + ], + selection_args=( + ["-url", str(entry_url)] if entry_url else ["-url", str(url), "-item", str(idx)] + ), + media_kind="playlist-item", + playlist_index=idx, + url=entry_url, + target=entry_url, + ) + results_list.append(row) + table.add_result(row) + + pipeline_context.set_current_stage_table(table) + pipeline_context.set_last_result_table(table, results_list) + try: + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + with cm: + get_stderr_console().print(table) + except Exception: + pass + setattr(table, "_rendered_by_cmdlet", True) + return True + + def _show_format_table( + self, + *, + url: str, + args: Sequence[str], + clip_spec: Optional[str], + clip_values: Sequence[str], + ytdlp_tool: YtDlpTool, + formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], + ) -> bool: + ctx = pipeline_context.get_stage_context() + if ctx is not None and getattr(ctx, "total_stages", 0) > 1: + return False + + formats = _list_formats_cached( + url, + playlist_items_value=None, + formats_cache=formats_cache, + ytdlp_tool=ytdlp_tool, + ) + if not formats or len(formats) <= 1: + return False + + candidate_formats = [f for f in formats if is_browseable_format(f)] + filtered_formats = candidate_formats if candidate_formats else list(formats) + base_cmd = f'download-file "{url}"' + remaining_args = [arg for arg in args if arg not in [url] and not str(arg).startswith("-")] + if remaining_args: + base_cmd += " " + " ".join(remaining_args) + + table = Table(title=f"Available formats for {url}", max_columns=10, preserve_order=True) + table.set_table("ytdlp.formatlist") + table.set_source_command("download-file", [url]) + + results_list: List[Dict[str, Any]] = [] + for idx, fmt in enumerate(filtered_formats, 1): + format_id = fmt.get("format_id", "") + selection_format_id = format_id + try: + if str(fmt.get("vcodec", "none")) != "none" and str(fmt.get("acodec", "none")) == "none" and format_id: + selection_format_id = f"{format_id}+bestaudio" + except Exception: + selection_format_id = format_id + + format_dict = format_for_table_selection( + fmt, + url, + idx, + selection_format_id=selection_format_id, + ) + format_dict["cmd"] = base_cmd + selection_args: List[str] = list(format_dict.get("_selection_args") or []) + if (not clip_spec) and clip_values: + clip_query = f"clip:{','.join([v for v in clip_values if v])}" + selection_args = _merge_query_args(selection_args, clip_query) + format_dict["_selection_args"] = selection_args + format_dict.setdefault("full_metadata", {})["_selection_args"] = selection_args + results_list.append(format_dict) + table.add_result(format_dict) + + try: + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + with cm: + get_stderr_console().print(table) + except Exception: + pass + + setattr(table, "_rendered_by_cmdlet", True) + pipeline_context.set_current_stage_table(table) + pipeline_context.set_last_result_table(table, results_list) + return True + + def download_url( + self, + url: str, + output_dir: Path, + **kwargs: Any, + ) -> Optional[Any]: + url_str = str(url or "").strip() + if not url_str or not is_url_supported_by_ytdlp(url_str): + return None + + parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {} + args = kwargs.get("args") if isinstance(kwargs.get("args"), list) else [] + progress = kwargs.get("progress") + quiet_mode = bool(kwargs.get("quiet_mode")) + + if progress is None: + try: + progress = self.config.get("_pipeline_progress") if isinstance(self.config, dict) else None + except Exception: + progress = None + if progress is None: + progress = PipelineProgress(pipeline_context) + + query_spec = parsed.get("query") + clip_spec = parsed.get("clip") + query_keyed = _parse_query_keyed_spec(str(query_spec) if query_spec is not None else None) + clip_values: List[str] = [] + item_values: List[str] = [] + if clip_spec: + keyed = _parse_keyed_csv_spec(str(clip_spec), default_key="clip") + clip_values.extend(keyed.get("clip", []) or []) + item_values.extend(keyed.get("item", []) or []) + if query_keyed: + clip_values.extend(query_keyed.get("clip", []) or []) + item_values.extend(query_keyed.get("item", []) or []) + + if item_values and not parsed.get("item"): + parsed["item"] = ",".join([v for v in item_values if v]) + + clip_ranges = None + if clip_values: + clip_ranges = _parse_time_ranges(",".join([v for v in clip_values if v])) + if not clip_ranges: + log(f"Invalid clip format: {clip_spec or query_spec}", file=sys.stderr) + return {"action": "handled", "exit_code": 1} + + ytdlp_tool = YtDlpTool(self.config) + formats_cache: Dict[str, Optional[List[Dict[str, Any]]]] = {} + playlist_items = str(parsed.get("item")) if parsed.get("item") else None + query_format: Optional[str] = None + try: + fmt_values = query_keyed.get("format", []) if isinstance(query_keyed, dict) else [] + fmt_candidate = fmt_values[-1] if fmt_values else None + if fmt_candidate is not None: + query_format = str(fmt_candidate).strip() + except Exception: + query_format = None + + query_audio: Optional[bool] = None + try: + audio_values = query_keyed.get("audio", []) if isinstance(query_keyed, dict) else [] + audio_candidate = audio_values[-1] if audio_values else None + if audio_candidate is not None: + s_val = str(audio_candidate).strip().lower() + if s_val in {"1", "true", "t", "yes", "y", "on"}: + query_audio = True + elif s_val in {"0", "false", "f", "no", "n", "off"}: + query_audio = False + elif s_val: + query_audio = True + except Exception: + query_audio = None + + query_wants_audio = bool(query_format and str(query_format).strip().lower() == "audio") + wants_audio = bool(query_audio) if query_audio is not None else bool(query_wants_audio) + mode = "audio" if wants_audio else "video" + + ytdl_format: Optional[str] = None + height_selector = None + if query_format and not query_wants_audio: + try: + height_selector = ytdlp_tool.resolve_height_selector(query_format) + except Exception: + height_selector = None + if query_wants_audio: + ytdl_format = "bestaudio" + elif height_selector: + ytdl_format = height_selector + elif query_format: + ytdl_format = query_format + + if not playlist_items: + if query_format and not query_wants_audio and not ytdl_format: + try: + idx_fmt = _format_id_for_query_index(query_format, url_str, formats_cache, ytdlp_tool) + if idx_fmt: + ytdl_format = idx_fmt + except ValueError as exc: + debug(f"[ytdlp] Format resolution for '{query_format}' failed ({exc}); treating as literal") + ytdl_format = query_format + + if not ytdl_format and self._show_playlist_table(url=url_str, ytdlp_tool=ytdlp_tool): + return {"action": "handled", "exit_code": 0} + + if ( + mode != "audio" + and not clip_spec + and not clip_values + and not playlist_items + and not ytdl_format + and self._show_format_table( + url=url_str, + args=args, + clip_spec=str(clip_spec) if clip_spec is not None else None, + clip_values=clip_values, + ytdlp_tool=ytdlp_tool, + formats_cache=formats_cache, + ) + ): + return {"action": "handled", "exit_code": 0} + + if mode == "video" and not ytdl_format and not query_format and not query_wants_audio: + try: + fmts = _list_formats_cached( + url_str, + playlist_items_value=playlist_items, + formats_cache=formats_cache, + ytdlp_tool=ytdlp_tool, + ) + if fmts: + has_video = any(str(f.get("vcodec", "none")) != "none" for f in fmts if isinstance(f, dict)) + has_audio = any(str(f.get("acodec", "none")) != "none" for f in fmts if isinstance(f, dict)) + if has_audio and not has_video: + mode = "audio" + ytdl_format = ytdlp_tool.default_format("audio") + elif "bandcamp.com/album/" in url_str: + mode = "audio" + ytdl_format = ytdlp_tool.default_format("audio") + except Exception as exc: + debug(f"[ytdlp] Audio-only detection error: {exc}") + + if mode == "audio" and not ytdl_format: + ytdl_format = "bestaudio" + if mode == "video" and not ytdl_format: + configured = (ytdlp_tool.default_format("video") or "").strip() + if configured and configured != "bestvideo+bestaudio/best": + resolved = ytdlp_tool.resolve_height_selector(configured) + ytdl_format = resolved or configured + + clip_sections_spec = _build_clip_sections_spec(clip_ranges) + if clip_sections_spec and mode != "audio": + clip_format_basis = ytdl_format + if not clip_format_basis or str(clip_format_basis).strip().lower() in { + "bestvideo+bestaudio/best", + "bestvideo+bestaudio", + "best", + "best/b", + "best/best", + "b", + }: + preferred_clip_format = str(getattr(ytdlp_tool.defaults, "format", "") or "").strip() + if preferred_clip_format and preferred_clip_format.lower() != "audio": + clip_format_basis = preferred_clip_format + else: + clip_format_basis = ytdlp_tool.default_format("video") + clip_safe_format = ytdlp_tool.resolve_clip_safe_format(clip_format_basis) + if clip_safe_format: + ytdl_format = clip_safe_format + + timeout_seconds = 300 + try: + override = self.config.get("_pipeobject_timeout_seconds") if isinstance(self.config, dict) else None + if override is not None: + timeout_seconds = max(1, int(override)) + except Exception: + timeout_seconds = 300 + + actual_format = ytdl_format + actual_playlist_items = playlist_items + if playlist_items and not ytdl_format and re.search(r"[^0-9,-]", playlist_items): + actual_format = playlist_items + actual_playlist_items = None + + attempted_single_format_fallback = False + attempted_audio_fallback_specific = False + attempted_audio_fallback_generic = False + while True: + try: + opts = DownloadOptions( + url=url_str, + mode=mode, + output_dir=output_dir, + ytdl_format=actual_format, + cookies_path=ytdlp_tool.resolve_cookiefile(), + clip_sections=clip_sections_spec, + playlist_items=actual_playlist_items, + quiet=quiet_mode, + no_playlist=False, + embed_chapters=True, + write_sub=True, + ) + result_obj = _download_with_timeout(opts, timeout_seconds=timeout_seconds, config=self.config) + break + except DownloadError as exc: + cause = getattr(exc, "__cause__", None) + detail = str(cause or "") + msg_lc = str(exc or "").lower() + detail_lc = detail.lower() + requested_format_unavailable = ( + "requested format is not available" in detail_lc + or "requested format is not available" in msg_lc + ) + + if requested_format_unavailable and mode == "audio": + if not attempted_audio_fallback_specific: + attempted_audio_fallback_specific = True + audio_format_id = None + try: + formats = _list_formats_cached( + url_str, + playlist_items_value=actual_playlist_items, + formats_cache=formats_cache, + ytdlp_tool=ytdlp_tool, + ) + if formats: + audio_candidates = [] + for fmt in formats: + if not isinstance(fmt, dict): + continue + vcodec = str(fmt.get("vcodec", "none")) + acodec = str(fmt.get("acodec", "none")) + if acodec != "none" and vcodec == "none": + audio_candidates.append(fmt) + if audio_candidates: + def _score_audio(fmt: Dict[str, Any]) -> float: + score = 0.0 + fid = str(fmt.get("format_id") or "").lower() + if "drc" in fid: + score -= 1000.0 + for key in ("abr", "tbr", "filesize", "filesize_approx"): + val = fmt.get(key) + if isinstance(val, (int, float)): + score += float(val) + break + if isinstance(val, str) and val.strip().isdigit(): + score += float(val) + break + return score + audio_candidates.sort(key=_score_audio, reverse=True) + audio_format_id = str(audio_candidates[0].get("format_id") or "").strip() or None + except Exception: + audio_format_id = None + if audio_format_id: + actual_format = audio_format_id + continue + + if not attempted_audio_fallback_generic and actual_format != "bestaudio/best": + attempted_audio_fallback_generic = True + actual_format = "bestaudio/best" + continue + + if requested_format_unavailable and mode != "audio": + formats = _list_formats_cached( + url_str, + playlist_items_value=actual_playlist_items, + formats_cache=formats_cache, + ytdlp_tool=ytdlp_tool, + ) + if ( + (not attempted_single_format_fallback) + and isinstance(formats, list) + and len(formats) == 1 + and isinstance(formats[0], dict) + ): + only = formats[0] + fallback_format = str(only.get("format_id") or "").strip() + selection_format_id = fallback_format + try: + vcodec = str(only.get("vcodec", "none")) + acodec = str(only.get("acodec", "none")) + if not clip_sections_spec and vcodec != "none" and acodec == "none" and fallback_format: + selection_format_id = f"{fallback_format}+bestaudio" + except Exception: + selection_format_id = fallback_format + if selection_format_id: + attempted_single_format_fallback = True + actual_format = selection_format_id + continue + + if isinstance(formats, list) and formats: + table = Table(title=f"Available formats for {url_str}", max_columns=10, preserve_order=True) + table.set_table("ytdlp.formatlist") + table.set_source_command("download-file", [url_str]) + results_list: List[Dict[str, Any]] = [] + for idx, fmt in enumerate(formats, 1): + format_id = str(fmt.get("format_id") or "") + selection_format_id = format_id + try: + if str(fmt.get("vcodec", "none")) != "none" and str(fmt.get("acodec", "none")) == "none" and format_id: + selection_format_id = f"{format_id}+bestaudio" + except Exception: + selection_format_id = format_id + size_str = "" + size_bytes = fmt.get("filesize") or fmt.get("filesize_approx") + try: + if isinstance(size_bytes, (int, float)) and size_bytes > 0: + size_str = f"{float(size_bytes) / (1024 * 1024):.1f}MB" + except Exception: + size_str = "" + format_dict = build_table_result_payload( + table="download-file", + title=f"Format {format_id}", + detail=" | ".join([part for part in [fmt.get("resolution", ""), fmt.get("ext", ""), size_str] if part]), + columns=[ + ("ID", format_id), + ("Resolution", str(fmt.get("resolution") or "N/A")), + ("Ext", str(fmt.get("ext") or "")), + ("Size", size_str), + ("Video", str(fmt.get("vcodec") or "none")), + ("Audio", str(fmt.get("acodec") or "none")), + ], + selection_args=["-query", f"format:{selection_format_id}"], + url=url_str, + target=url_str, + media_kind="format", + full_metadata={ + "format_id": format_id, + "url": url_str, + "item_selector": selection_format_id, + }, + ) + results_list.append(format_dict) + table.add_result(format_dict) + + pipeline_context.set_current_stage_table(table) + pipeline_context.set_last_result_table(table, results_list) + try: + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + with cm: + get_stderr_console().print(table) + except Exception: + pass + log("Requested format is not available; select a working format with @N", file=sys.stderr) + return {"action": "handled", "exit_code": 1} + + log(f"Download failed for {url_str}: {exc}", file=sys.stderr) + return {"action": "handled", "exit_code": 1} + except Exception as exc: + log(f"Error processing {url_str}: {exc}", file=sys.stderr) + return {"action": "handled", "exit_code": 1} + + pipe_objects = _build_pipe_objects( + result_obj, + url=url_str, + opts=opts, + embed_chapters=True, + write_sub=True, + ) + if clip_ranges and len(pipe_objects) == len(clip_ranges): + _apply_clip_decorations(pipe_objects, clip_ranges) + return {"action": "emit_pipe_objects", "items": pipe_objects, "exit_code": 0} + + def download_url_as_pipe_objects( + self, + url: str, + *, + output_dir: Optional[Path] = None, + mode_hint: Optional[str] = None, + ytdl_format_hint: Optional[str] = None, + ) -> List[Dict[str, Any]]: + url_str = str(url or "").strip() + if not url_str or not is_url_supported_by_ytdlp(url_str): + return [] + + out_dir = output_dir + if out_dir is None: + try: + from SYS.config import resolve_output_dir + out_dir = resolve_output_dir(self.config) + except Exception: + out_dir = None + if out_dir is None: + return [] + + mode = str(mode_hint or "").strip().lower() if mode_hint else "" + if mode not in {"audio", "video"}: + mode = "video" + try: + fmts_probe = list_formats( + url_str, + no_playlist=False, + playlist_items=None, + cookiefile=_cookiefile_str(YtDlpTool(self.config)), + ) + if isinstance(fmts_probe, list) and fmts_probe: + has_video = any( + str(f.get("vcodec", "none") or "none").strip().lower() != "none" + for f in fmts_probe + if isinstance(f, dict) + ) + mode = "video" if has_video else "audio" + except Exception: + mode = "video" + + chosen_format = str(ytdl_format_hint).strip() if ytdl_format_hint else None + if not chosen_format and mode == "audio": + chosen_format = "bestaudio" + + quiet_download = False + try: + quiet_download = bool((self.config or {}).get("_quiet_background_output")) + except Exception: + quiet_download = False + + opts = DownloadOptions( + url=url_str, + mode=mode, + output_dir=Path(out_dir), + cookies_path=YtDlpTool(self.config).resolve_cookiefile(), + ytdl_format=chosen_format, + quiet=quiet_download, + embed_chapters=True, + write_sub=True, + ) + try: + result_obj = _download_with_timeout(opts, timeout_seconds=300, config=self.config) + except Exception as exc: + log(f"[ytdlp] Download failed for {url_str}: {exc}", file=sys.stderr) + return [] + return _build_pipe_objects( + result_obj, + url=url_str, + opts=opts, + embed_chapters=True, + write_sub=True, + ) + -# Minimal provider registration for the new table system try: - from SYS.result_table_adapters import register_provider + from SYS.result_table_adapters import register_plugin from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column def _convert_format_result_to_model(sr: Any) -> ResultModel: - """Convert format dict to ResultModel for strict table display.""" d = sr.to_dict() if hasattr(sr, "to_dict") else (sr if isinstance(sr, dict) else {}) title = d.get("title") or f"Format {d.get('format_id', 'unknown')}" - - # Extract metadata from columns and full_metadata metadata: Dict[str, Any] = {} columns = d.get("columns") or [] for name, value in columns: key = str(name or "").strip().lower() if key in ("id", "resolution", "ext", "size", "video", "audio", "format_id"): metadata[key] = value - try: fm = d.get("full_metadata") or {} if isinstance(fm, dict): - for k, v in fm.items(): - metadata[str(k).strip().lower()] = v + for key, value in fm.items(): + metadata[str(key).strip().lower()] = value except Exception: pass - return ResultModel( title=str(title), path=d.get("url") or d.get("target"), ext=d.get("ext"), size_bytes=None, metadata=metadata, - source="ytdlp" + source="ytdlp", ) def _adapter(items: Iterable[Any]) -> Iterable[ResultModel]: - """Adapter to convert format results to ResultModels.""" - for it in items: + for item in items: try: - yield _convert_format_result_to_model(it) + yield _convert_format_result_to_model(item) except Exception: continue def _has_metadata(rows: List[ResultModel], key: str) -> bool: - """Check if any row has a given metadata key with a non-empty value.""" for row in rows: md = row.metadata or {} if key in md: @@ -256,7 +1343,6 @@ try: return False def _columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: - """Build column specifications from available metadata in rows.""" cols = [title_column()] if _has_metadata(rows, "resolution"): cols.append(metadata_column("resolution", "Resolution")) @@ -271,81 +1357,61 @@ try: return cols def _selection_fn(row: ResultModel) -> List[str]: - """Return selection args for format selection. - - When user selects @N, these args are passed to download-file which sees - the format query and skips the format table, downloading directly. - """ metadata = row.metadata or {} - - # Check for explicit selection args first args = metadata.get("_selection_args") or metadata.get("selection_args") if isinstance(args, (list, tuple)) and args: result_args = [str(x) for x in args if x is not None] debug(f"[ytdlp] Selection routed with args: {result_args}") return result_args - - # Fallback: use format_id format_id = metadata.get("format_id") or metadata.get("id") if format_id: result_args = ["-query", f"format:{format_id}"] debug(f"[ytdlp] Selection routed with format_id: {format_id}") return result_args - - debug("[ytdlp] Warning: No selection args or format_id found in row") return [] - register_provider( + register_plugin( "ytdlp.formatlist", _adapter, columns=_columns_factory, selection_fn=_selection_fn, metadata={"description": "ytdlp format selector for streaming media"}, ) - debug("[ytdlp] Provider registered successfully with TABLE_AUTO_STAGES routing to download-file") - # Also register the search table def _convert_search_result_to_model(sr: Any) -> ResultModel: - """Convert YouTube SearchResult to ResultModel for strict table display.""" d = sr.to_dict() if hasattr(sr, "to_dict") else (sr if isinstance(sr, dict) else {"title": getattr(sr, "title", str(sr))}) title = d.get("title") or "" path = d.get("path") or None columns = d.get("columns") or getattr(sr, "columns", None) or [] - - # Extract metadata from columns and full_metadata metadata: Dict[str, Any] = {} for name, value in columns: key = str(name or "").strip().lower() if key in ("uploader", "duration", "views", "video_id"): metadata[key] = value - try: fm = d.get("full_metadata") or {} if isinstance(fm, dict): - for k, v in fm.items(): - metadata[str(k).strip().lower()] = v + for key, value in fm.items(): + metadata[str(key).strip().lower()] = value except Exception: pass - return ResultModel( title=str(title), path=str(path) if path else None, ext=None, size_bytes=None, metadata=metadata, - source="ytdlp" + source="ytdlp", ) def _search_adapter(items: Iterable[Any]) -> Iterable[ResultModel]: - """Adapter to convert search results to ResultModels.""" - for it in items: + for item in items: try: - yield _convert_search_result_to_model(it) + yield _convert_search_result_to_model(item) except Exception: continue def _search_columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: - """Build column specifications for search results.""" cols = [title_column()] if _has_metadata(rows, "uploader"): cols.append(metadata_column("uploader", "Uploader")) @@ -356,32 +1422,20 @@ try: return cols def _search_selection_fn(row: ResultModel) -> List[str]: - """Return selection args for search results. - - When user selects @N from search results, route to download-file with -url. - """ metadata = row.metadata or {} - - # Check for explicit selection args first args = metadata.get("_selection_args") or metadata.get("selection_args") if isinstance(args, (list, tuple)) and args: return [str(x) for x in args if x is not None] - - # Fallback to direct URL if row.path: return ["-url", row.path] - return ["-title", row.title or ""] - register_provider( + register_plugin( "ytdlp.search", _search_adapter, columns=_search_columns_factory, selection_fn=_search_selection_fn, metadata={"description": "ytdlp video search using yt-dlp"}, ) - debug("[ytdlp] Search provider registered successfully") -except Exception as e: - # best-effort registration - debug(f"[ytdlp] Provider registration note: {e}") - pass +except Exception as exc: + debug(f"[ytdlp] Provider registration note: {exc}") diff --git a/Provider/zeroxzero.py b/Provider/zeroxzero.py index 6ad07b1..18a3f9d 100644 --- a/Provider/zeroxzero.py +++ b/Provider/zeroxzero.py @@ -11,8 +11,8 @@ from SYS.logger import log class ZeroXZero(Provider): """File provider for 0x0.st.""" - NAME = "0x0" - PROVIDER_ALIASES = ("zeroxzero",) + PLUGIN_NAME = "0x0" + PLUGIN_ALIASES = ("zeroxzero",) def upload(self, file_path: str, **kwargs: Any) -> str: from API.HTTP import HTTPClient diff --git a/ProviderCore/__init__.py b/ProviderCore/__init__.py index 55cf273..2f23cef 100644 --- a/ProviderCore/__init__.py +++ b/ProviderCore/__init__.py @@ -1,5 +1,6 @@ -"""Provider core modules. +"""Plugin core modules. -This package contains the provider framework (base types, registry, and shared helpers). -Concrete provider implementations live in the `Provider/` package. +This package contains the plugin framework (base types, registry, and shared +helpers). Built-in plugins continue to live in the `Provider/` package for +backward compatibility. """ diff --git a/ProviderCore/base.py b/ProviderCore/base.py index 26ea6a2..214aaf9 100644 --- a/ProviderCore/base.py +++ b/ProviderCore/base.py @@ -10,9 +10,9 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple, Callable @dataclass class SearchResult: - """Unified search result format across all search providers.""" + """Unified search result format across all search plugins.""" - table: str # Provider name: "libgen", "soulseek", "bandcamp", "youtube", etc. + table: str # Plugin name: "libgen", "soulseek", "bandcamp", "youtube", etc. title: str # Display title/filename path: str # Download target (URL, path, magnet, identifier) @@ -84,7 +84,7 @@ class SearchResult: def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]: - """Extract inline key:value arguments from a provider search query.""" + """Extract inline key:value arguments from a plugin search query.""" query_text = str(raw_query or "").strip() if not query_text: @@ -112,10 +112,10 @@ def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]: class Provider(ABC): - """Unified provider base class. + """Unified plugin base class. - This replaces the older split between "search providers" and "file providers". - Concrete providers may implement any subset of: + This replaces the older split between search and upload providers. + Concrete plugins may implement any subset of: - search(query, ...) - download(result, output_dir) - upload(file_path, ...) @@ -124,7 +124,8 @@ class Provider(ABC): """ URL: Sequence[str] = () - NAME: str = "" + PLUGIN_NAME: str = "" + PLUGIN_ALIASES: Sequence[str] = () # Optional provider-driven defaults for what to do when a user selects @N from a # provider table. The CLI uses this to auto-insert stages (e.g. download-file) @@ -141,24 +142,23 @@ class Provider(ABC): # Used for dynamically generating config panels (e.g., missing credentials). REQUIRED_CONFIG_KEYS: Sequence[str] = () - # Some providers implement `upload()` but are not intended to be used as - # generic "file host" providers via `add-file -provider ...`. + # Some plugins implement `upload()` but are not intended to be used as + # generic "file host" plugins via `add-file -plugin ...`. EXPOSE_AS_FILE_PROVIDER: bool = True def __init__(self, config: Optional[Dict[str, Any]] = None): self.config = config or {} - # Prioritize explicit NAME property for the instance name self.name = str( - getattr(self, "NAME", None) - or getattr(self, "PROVIDER_NAME", None) + getattr(self, "PLUGIN_NAME", None) or self.__class__.__name__ ).lower() @property def label(self) -> str: - """Friendly display name for the provider.""" - if hasattr(self, "NAME") and self.NAME: - name = str(self.NAME) + """Friendly display name for the plugin.""" + name = str(getattr(self, "PLUGIN_NAME", None) or self.__class__.__name__) + + if name: if name.lower() == "loc": return "LoC" if name.lower() == "openlibrary": @@ -186,7 +186,7 @@ class Provider(ABC): def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Return metadata for the results table.""" - return {"provider": self.name} + return {"plugin": self.name} def get_source_command(self, args_list: List[str]) -> Tuple[str, List[str]]: """Return the command and arguments that produced this search result. @@ -308,6 +308,49 @@ class Provider(ABC): _ = config return 0 + def resolve_pipe_result_download( + self, + result: Any, + pipe_obj: Any, + ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: + """Materialize a piped plugin result into a local file for add-file.""" + + _ = result + _ = pipe_obj + return None, None, None + + def expand_selection( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + table_type: str = "", + **_kwargs: Any, + ) -> Optional[List[Any]]: + """Optionally expand a selection into downstream items for non-terminal pipelines.""" + + _ = selected_items + _ = ctx + _ = stage_is_last + _ = table_type + return None + + def status_summary(self) -> Dict[str, Any]: + """Return plugin-owned status details for startup/status views.""" + + enabled = False + try: + enabled = bool(self.validate()) + except Exception: + enabled = False + return { + "status": "ENABLED" if enabled else "DISABLED", + "name": self.label, + "plugin": self.name, + "detail": "Configured" if enabled else "Not configured", + } + def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]: """Optional provider override to parse and act on URLs.""" @@ -315,6 +358,67 @@ class Provider(ABC): _ = output_dir return False, None + def download_url(self, url: str, output_dir: Path, **_kwargs: Any) -> Optional[Any]: + """Optional direct-URL download hook used by generic cmdlets.""" + + _ = url + _ = output_dir + return None + + def resolve_url(self, url: str, **_kwargs: Any) -> str: + """Optionally normalize or exchange a URL before downstream use.""" + + return str(url or "") + + def resolve_playback_path(self, item: Any, **_kwargs: Any) -> Optional[str]: + """Optionally turn a plugin-owned item into a playable local path or URL.""" + + _ = item + return None + + def list_url_formats(self, url: str, **_kwargs: Any) -> Optional[List[Dict[str, Any]]]: + """Optionally return picker-friendly format metadata for a URL.""" + + _ = url + return None + + def filter_picker_formats( + self, + formats: List[Dict[str, Any]], + **_kwargs: Any, + ) -> List[Dict[str, Any]]: + """Optionally filter or reorder raw format rows before UI display.""" + + return list(formats or []) + + def enrich_playlist_entries( + self, + entries: List[Dict[str, Any]], + **_kwargs: Any, + ) -> Optional[List[Dict[str, Any]]]: + """Optionally expand lightweight playlist entries with richer metadata.""" + + _ = entries + return None + + def maybe_show_picker( + self, + *, + url: str, + item: Optional[Any] = None, + parsed: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, + quiet_mode: bool = False, + ) -> Optional[int]: + """Optional hook for plugins that want to render an interactive picker/table.""" + + _ = url + _ = item + _ = parsed + _ = config + _ = quiet_mode + return None + def upload(self, file_path: str, **kwargs: Any) -> str: """Upload a file and return a URL or identifier.""" raise NotImplementedError(f"Provider '{self.name}' does not support upload") @@ -419,6 +523,25 @@ class Provider(ABC): patterns.append(candidate) return tuple(patterns) + @classmethod + def selection_url_prefixes(cls) -> Tuple[str, ...]: + """Return URL-like prefixes that selection parsing should treat as URLs.""" + + prefixes: List[str] = [] + seen: set[str] = set() + for pattern in cls.url_patterns(): + try: + candidate = str(pattern or "").strip().lower() + except Exception: + continue + if not candidate: + continue + if "://" in candidate or candidate.endswith(":") or "🧲" in candidate: + if candidate not in seen: + seen.add(candidate) + prefixes.append(candidate) + return tuple(prefixes) + class SearchProvider(Provider): """Compatibility alias for older code. diff --git a/ProviderCore/registry.py b/ProviderCore/registry.py index ebc7c5b..41543fc 100644 --- a/ProviderCore/registry.py +++ b/ProviderCore/registry.py @@ -1,17 +1,21 @@ -"""Provider registry. +"""Plugin registry. -Concrete provider implementations live in the ``Provider`` package. This module -is the single source of truth for discovery, metadata, and lifecycle helpers -for those plugins. +Built-in plugin implementations live in the ``Provider`` package. External user +plugins can be dropped into a repo-local ``plugins/`` directory or discovered +via environment-configured plugin paths. """ from __future__ import annotations from functools import lru_cache +import hashlib import importlib +import importlib.util +import os import pkgutil import sys from dataclasses import dataclass, field +from pathlib import Path from types import ModuleType from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Type from urllib.parse import urlparse @@ -21,21 +25,85 @@ from SYS.logger import log, debug from ProviderCore.base import FileProvider, Provider, SearchProvider, SearchResult -def download_soulseek_file(*args: Any, **kwargs: Any) -> Any: - """Lazy proxy for the soulseek downloader. +_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH") - Importing the provider modules can be expensive; keeping this lazy avoids - paying that cost at registry import time. - """ - from Provider.soulseek import download_soulseek_file as _download +def _repo_root() -> Path: + try: + return Path(__file__).resolve().parents[1] + except Exception: + return Path.cwd() - return _download(*args, **kwargs) +def _iter_external_plugin_dirs() -> Tuple[Path, ...]: + seen: set[str] = set() + dirs: List[Path] = [] + + candidates: List[Path] = [_repo_root() / "plugins"] + try: + cwd_plugins = Path.cwd() / "plugins" + if cwd_plugins not in candidates: + candidates.append(cwd_plugins) + except Exception: + pass + + for env_name in _EXTERNAL_PLUGIN_ENV_VARS: + raw_value = str(os.environ.get(env_name, "") or "").strip() + if not raw_value: + continue + for chunk in raw_value.split(os.pathsep): + text = str(chunk or "").strip().strip('"') + if not text: + continue + candidates.append(Path(text).expanduser()) + + for candidate in candidates: + try: + resolved = candidate.resolve() + except Exception: + resolved = candidate + key = str(resolved).lower() + if key in seen: + continue + seen.add(key) + try: + if resolved.exists() and resolved.is_dir(): + dirs.append(resolved) + except Exception: + continue + + return tuple(dirs) + + +def _iter_external_plugin_entries(plugin_dir: Path) -> Iterable[Tuple[str, Path, bool]]: + try: + children = sorted(plugin_dir.iterdir(), key=lambda entry: entry.name.lower()) + except Exception: + return () + + out: List[Tuple[str, Path, bool]] = [] + for child in children: + name = str(child.name or "").strip() + if not name or name.startswith("."): + continue + + if child.is_file() and child.suffix.lower() == ".py" and child.stem != "__init__": + fingerprint = hashlib.sha1(str(child).encode("utf-8", errors="ignore")).hexdigest()[:10] + out.append((f"_medeia_plugin_{child.stem}_{fingerprint}", child, False)) + continue + + if child.is_dir(): + init_py = child / "__init__.py" + if not init_py.exists() or not init_py.is_file(): + continue + fingerprint = hashlib.sha1(str(child).encode("utf-8", errors="ignore")).hexdigest()[:10] + out.append((f"_medeia_plugin_pkg_{child.name}_{fingerprint}", init_py, True)) + + return tuple(out) @dataclass(frozen=True) class ProviderInfo: - """Metadata about a single provider entry.""" + """Metadata about a single plugin entry.""" canonical_name: str provider_class: Type[Provider] @@ -56,14 +124,16 @@ class ProviderInfo: class ProviderRegistry: - """Handles discovery, registration, and lookup of provider classes.""" + """Handles discovery, registration, and lookup of built-in and external plugins.""" def __init__(self, package_name: str) -> None: self.package_name = (package_name or "").strip() self._infos: Dict[str, ProviderInfo] = {} self._lookup: Dict[str, ProviderInfo] = {} self._modules: set[str] = set() + self._external_modules: set[str] = set() self._discovered = False + self._external_dirs_scanned = False def _normalize(self, value: Any) -> str: return str(value or "").strip().lower() @@ -85,12 +155,10 @@ class ProviderRegistry: if override_name: _add(override_name) else: - # Use explicit NAME or PROVIDER_NAME if available, else class name - _add(getattr(provider_class, "NAME", None)) - _add(getattr(provider_class, "PROVIDER_NAME", None)) + _add(getattr(provider_class, "PLUGIN_NAME", None)) _add(getattr(provider_class, "__name__", None)) - for alias in getattr(provider_class, "PROVIDER_ALIASES", ()) or (): + for alias in getattr(provider_class, "PLUGIN_ALIASES", ()) or (): _add(alias) return names @@ -104,14 +172,14 @@ class ProviderRegistry: module_name: Optional[str] = None, replace: bool = False, ) -> ProviderInfo: - """Register a provider class with canonical and alias names.""" + """Register a plugin class with canonical and alias names.""" candidates = self._candidate_names(provider_class, override_name) if not candidates: - raise ValueError("provider name candidates are required") + raise ValueError("plugin name candidates are required") canonical = self._normalize(candidates[0]) if not canonical: - raise ValueError("provider name must not be empty") + raise ValueError("plugin name must not be empty") alias_names: List[str] = [] alias_seen: set[str] = set() @@ -165,7 +233,44 @@ class ProviderRegistry: try: self.register(candidate, module_name=module_name) except Exception as exc: - log(f"[provider] Failed to register {module_name}.{candidate.__name__}: {exc}", file=sys.stderr) + log(f"[plugin] Failed to register {module_name}.{candidate.__name__}: {exc}", file=sys.stderr) + + def _discover_external_plugins(self) -> None: + if self._external_dirs_scanned: + return + self._external_dirs_scanned = True + + for plugin_dir in _iter_external_plugin_dirs(): + try: + plugin_dir_str = str(plugin_dir) + if plugin_dir_str and plugin_dir_str not in sys.path: + sys.path.insert(0, plugin_dir_str) + except Exception: + pass + + for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir): + if module_name in self._external_modules: + continue + + try: + if is_package: + spec = importlib.util.spec_from_file_location( + module_name, + str(module_path), + submodule_search_locations=[str(module_path.parent)], + ) + else: + spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + if spec is None or spec.loader is None: + raise ImportError("missing module spec loader") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + self._external_modules.add(module_name) + self._register_module(module) + except Exception as exc: + log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr) def discover(self) -> None: """Import and register providers from the package.""" @@ -177,12 +282,13 @@ class ProviderRegistry: try: package = importlib.import_module(self.package_name) except Exception as exc: - log(f"[provider] Failed to import package {self.package_name}: {exc}", file=sys.stderr) + log(f"[plugin] Failed to import package {self.package_name}: {exc}", file=sys.stderr) return self._register_module(package) package_path = getattr(package, "__path__", None) if not package_path: + self._discover_external_plugins() return for finder, module_name, _ in pkgutil.iter_modules(package_path): @@ -194,18 +300,19 @@ class ProviderRegistry: try: module = importlib.import_module(module_path) except Exception as exc: - log(f"[provider] Failed to load {module_path}: {exc}", file=sys.stderr) + log(f"[plugin] Failed to load {module_path}: {exc}", file=sys.stderr) continue self._register_module(module) # Pick up any Provider subclasses loaded via other mechanisms. self._sync_subclasses() + self._discover_external_plugins() def _try_import_for_name(self, normalized_name: str) -> None: - """Best-effort import for a single provider module. + """Best-effort import for a single plugin module. This avoids importing every provider module when the caller only needs - one provider (common for CLI usage). + one plugin (common for CLI usage). """ name = str(normalized_name or "").strip().lower() if not name or not self.package_name: @@ -249,6 +356,7 @@ class ProviderRegistry: # module that matches the requested name. if not self._discovered: self._try_import_for_name(normalized) + self._discover_external_plugins() info = self._lookup.get(normalized) if info is not None: return info @@ -279,6 +387,9 @@ class ProviderRegistry: _walk(Provider) REGISTRY = ProviderRegistry("Provider") +PLUGIN_REGISTRY = REGISTRY +PluginInfo = ProviderInfo +PluginRegistry = ProviderRegistry @lru_cache(maxsize=512) @@ -289,18 +400,16 @@ def _provider_url_patterns(provider_class: Type[Provider]) -> Sequence[str]: return [] -def register_provider( - provider_class: Type[Provider], +def register_plugin( + plugin_class: Type[Provider], *, name: Optional[str] = None, aliases: Optional[Sequence[str]] = None, module_name: Optional[str] = None, replace: bool = False, ) -> ProviderInfo: - """Register a provider class from tests or third-party packages.""" - return REGISTRY.register( - provider_class, + plugin_class, override_name=name, extra_aliases=aliases, module_name=module_name, @@ -308,7 +417,7 @@ def register_provider( ) -def get_provider_class(name: str) -> Optional[Type[Provider]]: +def get_plugin_class(name: str) -> Optional[Type[Provider]]: info = REGISTRY.get(name) if info is None: return None @@ -323,18 +432,18 @@ def selection_auto_stage_for_table( if not t: return None - provider_key = t.split(".", 1)[0] if "." in t else t - provider_class = get_provider_class(provider_key) or get_provider_class(t) - if provider_class is None: + plugin_key = t.split(".", 1)[0] if "." in t else t + plugin_class = get_plugin_class(plugin_key) or get_plugin_class(t) + if plugin_class is None: return None try: - return provider_class.selection_auto_stage(t, stage_args) + return plugin_class.selection_auto_stage(t, stage_args) except Exception: return None -def is_known_provider_name(name: str) -> bool: +def is_known_plugin_name(name: str) -> bool: return REGISTRY.has_name(name) @@ -406,83 +515,83 @@ def _collect_inline_choice_mapping(provider: Provider) -> Dict[str, List[Dict[st return mapping -def get_provider(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[Provider]: +def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[Provider]: info = REGISTRY.get(name) if info is None: - debug(f"[provider] Unknown provider: {name}") + debug(f"[plugin] Unknown plugin: {name}") return None try: - provider = info.provider_class(config) - if not provider.validate(): - debug(f"[provider] Provider '{name}' is not available") + plugin = info.provider_class(config) + if not plugin.validate(): + debug(f"[plugin] Plugin '{name}' is not available") return None - return provider + return plugin except Exception as exc: - debug(f"[provider] Error initializing '{name}': {exc}") + debug(f"[plugin] Error initializing '{name}': {exc}") return None -def list_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: +def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: availability: Dict[str, bool] = {} for info in REGISTRY.iter_providers(): try: - provider = info.provider_class(config) - availability[info.canonical_name] = provider.validate() + plugin = info.provider_class(config) + availability[info.canonical_name] = plugin.validate() except Exception: availability[info.canonical_name] = False return availability -def get_search_provider(name: str, - config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]: - provider = get_provider(name, config) - if provider is None: +def get_search_plugin(name: str, + config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]: + plugin = get_plugin(name, config) + if plugin is None: return None - if not _supports_search(provider): - debug(f"[provider] Provider '{name}' does not support search") + if not _supports_search(plugin): + debug(f"[plugin] Plugin '{name}' does not support search") return None - return provider # type: ignore[return-value] + return plugin # type: ignore[return-value] -def list_search_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: +def list_search_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: availability: Dict[str, bool] = {} for info in REGISTRY.iter_providers(): try: - provider = info.provider_class(config) + plugin = info.provider_class(config) availability[info.canonical_name] = bool( - provider.validate() and info.supports_search + plugin.validate() and info.supports_search ) except Exception: availability[info.canonical_name] = False return availability -def get_file_provider(name: str, +def get_upload_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[FileProvider]: - provider = get_provider(name, config) - if provider is None: + plugin = get_plugin(name, config) + if plugin is None: return None - if not _supports_upload(provider): - debug(f"[provider] Provider '{name}' does not support upload") + if not _supports_upload(plugin): + debug(f"[plugin] Plugin '{name}' does not support upload") return None - return provider # type: ignore[return-value] + return plugin # type: ignore[return-value] -def list_file_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: +def list_upload_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: availability: Dict[str, bool] = {} for info in REGISTRY.iter_providers(): try: - provider = info.provider_class(config) + plugin = info.provider_class(config) availability[info.canonical_name] = bool( - provider.validate() and info.supports_upload + plugin.validate() and info.supports_upload ) except Exception: availability[info.canonical_name] = False return availability -def match_provider_name_for_url(url: str) -> Optional[str]: +def match_plugin_name_for_url(url: str) -> Optional[str]: raw_url = str(url or "").strip() raw_url_lower = raw_url.lower() try: @@ -540,31 +649,31 @@ def match_provider_name_for_url(url: str) -> Optional[str]: return None -def provider_inline_query_choices( - provider_name: str, +def plugin_inline_query_choices( + plugin_name: str, field_name: str, config: Optional[Dict[str, Any]] = None, ) -> List[str]: - """Return provider-declared inline query choices for a field (e.g., system:GBA). + """Return plugin-declared inline query choices for a field (e.g., system:GBA). - Providers can expose a mapping via ``QUERY_ARG_CHOICES`` (preferred) or + Plugins can expose a mapping via ``QUERY_ARG_CHOICES`` (preferred) or ``INLINE_QUERY_FIELD_CHOICES`` / ``inline_query_field_choices()``. The helper keeps completion logic simple and reusable. """ - pname = str(provider_name or "").strip().lower() + pname = str(plugin_name or "").strip().lower() field = str(field_name or "").strip().lower() if not pname or not field: return [] - provider = get_search_provider(pname, config) - if provider is None: - provider = get_provider(pname, config) - if provider is None: + plugin = get_search_plugin(pname, config) + if plugin is None: + plugin = get_plugin(pname, config) + if plugin is None: return [] try: - mapping = _collect_inline_choice_mapping(provider) + mapping = _collect_inline_choice_mapping(plugin) if not mapping: return [] @@ -593,12 +702,32 @@ def provider_inline_query_choices( return [] -def get_provider_for_url(url: str, - config: Optional[Dict[str, Any]] = None) -> Optional[Provider]: - name = match_provider_name_for_url(url) +def get_plugin_for_url(url: str, + config: Optional[Dict[str, Any]] = None) -> Optional[Provider]: + name = match_plugin_name_for_url(url) if not name: return None - return get_provider(name, config) + return get_plugin(name, config) + + +def list_selection_url_prefixes() -> List[str]: + prefixes: List[str] = [] + seen: set[str] = set() + for info in REGISTRY.iter_providers(): + try: + values = info.provider_class.selection_url_prefixes() + except Exception: + values = () + for value in values or (): + try: + normalized = str(value or "").strip().lower() + except Exception: + continue + if not normalized or normalized in seen: + continue + seen.add(normalized) + prefixes.append(normalized) + return prefixes def resolve_inline_filters( @@ -657,21 +786,25 @@ def resolve_inline_filters( __all__ = [ "ProviderInfo", + "PluginInfo", "Provider", "SearchProvider", "FileProvider", "SearchResult", - "register_provider", - "get_provider", - "list_providers", - "get_search_provider", - "list_search_providers", - "get_file_provider", - "list_file_providers", - "match_provider_name_for_url", - "get_provider_for_url", - "get_provider_class", + "PluginRegistry", + "PLUGIN_REGISTRY", + "register_plugin", + "get_plugin", + "list_plugins", + "get_search_plugin", + "list_search_plugins", + "get_upload_plugin", + "list_upload_plugins", + "match_plugin_name_for_url", + "get_plugin_for_url", + "list_selection_url_prefixes", + "get_plugin_class", "selection_auto_stage_for_table", - "download_soulseek_file", - "provider_inline_query_choices", + "plugin_inline_query_choices", + "is_known_plugin_name", ] diff --git a/SYS/cmdlet_catalog.py b/SYS/cmdlet_catalog.py index 4fc0b32..ea3158f 100644 --- a/SYS/cmdlet_catalog.py +++ b/SYS/cmdlet_catalog.py @@ -5,6 +5,7 @@ from importlib import import_module, reload as reload_module from types import ModuleType from typing import Any, Dict, List, Optional import logging +from ProviderCore.registry import get_plugin logger = logging.getLogger(__name__) try: @@ -370,22 +371,21 @@ def get_cmdlet_arg_choices( token = matrix_conf.get("access_token") if hs and token: try: - from Provider.matrix import Matrix - - try: - m = Matrix(config) - rooms = m.list_rooms(room_ids=ids) - choices = [] - for r in rooms or []: - name = str(r.get("name") or "").strip() - rid = str(r.get("room_id") or "").strip() - choices.append(name or rid) - if choices: - return choices - except Exception as exc: - logger.exception("Matrix provider failed while listing rooms: %s", exc) + provider = get_plugin("matrix", config) + if provider is not None: + try: + rooms = provider.list_rooms(room_ids=ids) + choices = [] + for r in rooms or []: + name = str(r.get("name") or "").strip() + rid = str(r.get("room_id") or "").strip() + choices.append(name or rid) + if choices: + return choices + except Exception as exc: + logger.exception("Matrix provider failed while listing rooms: %s", exc) except Exception as exc: - logger.exception("Failed to import Matrix provider or initialize: %s", exc) + logger.exception("Failed to initialize Matrix plugin: %s", exc) except Exception as exc: logger.exception("Failed to resolve matrix rooms: %s", exc) diff --git a/SYS/cmdlet_spec.py b/SYS/cmdlet_spec.py index 2eabef5..8699db6 100644 --- a/SYS/cmdlet_spec.py +++ b/SYS/cmdlet_spec.py @@ -90,10 +90,10 @@ class SharedArgs: description="http parser", ) - PROVIDER = CmdletArg( - name="provider", + PLUGIN = CmdletArg( + name="plugin", type="string", - description="selects provider", + description="selects plugin", ) @staticmethod @@ -284,7 +284,13 @@ class Cmdlet: return {f"-{arg_name}", f"--{arg_name}"} def build_flag_registry(self) -> Dict[str, set[str]]: - return {arg.name: self.get_flags(arg.name) for arg in self.arg} + registry: Dict[str, set[str]] = {} + for arg in self.arg: + try: + registry[arg.name] = {str(flag).lower() for flag in arg.to_flags()} + except Exception: + registry[arg.name] = {flag.lower() for flag in self.get_flags(arg.name)} + return registry def parse_cmdlet_args( @@ -335,8 +341,12 @@ def parse_cmdlet_args( positional_args.append(spec) arg_spec_map[canonical_key] = canonical_name - arg_spec_map[f"-{canonical_name}".lower()] = canonical_name - arg_spec_map[f"--{canonical_name}".lower()] = canonical_name + try: + for flag in spec.to_flags(): + arg_spec_map[str(flag).lower()] = canonical_name + except Exception: + arg_spec_map[f"-{canonical_name}".lower()] = canonical_name + arg_spec_map[f"--{canonical_name}".lower()] = canonical_name i = 0 positional_index = 0 diff --git a/SYS/metadata.py b/SYS/metadata.py index 81b17a3..9a79eea 100644 --- a/SYS/metadata.py +++ b/SYS/metadata.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple +from ProviderCore.registry import get_plugin from SYS.yt_metadata import extract_ytdlp_tags try: # Optional; used when available for richer metadata fetches @@ -2213,40 +2214,20 @@ def enrich_playlist_entries(entries: list, extractor: str) -> list: Returns: List of enriched entry dicts """ - # Import here to avoid circular dependency - from tool.ytdlp import is_url_supported_by_ytdlp - if not entries: return entries - enriched = [] - for entry in entries: - # If entry has a direct URL, fetch its full metadata - entry_url = entry.get("url") - if entry_url and is_url_supported_by_ytdlp(entry_url): - try: - import yt_dlp + plugin = get_plugin("ytdlp", {}) + if plugin is None: + return entries - ydl_opts: Any = { - "quiet": True, - "no_warnings": True, - "skip_download": True, - "noprogress": True, - "socket_timeout": 5, - "retries": 1, - } - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - full_info = ydl.extract_info(entry_url, download=False) - if full_info: - enriched.append(full_info) - continue - except Exception: - logger.exception("Failed to fetch full metadata for entry URL: %s", entry_url) + try: + enriched = plugin.enrich_playlist_entries(entries, extractor=extractor) + except Exception: + logger.exception("Failed to enrich playlist entries for extractor: %s", extractor) + return entries - # Fallback to original entry if fetch failed - enriched.append(entry) - - return enriched + return enriched if isinstance(enriched, list) else entries def format_playlist_entry(entry: Dict[str, diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 545edbd..54e19b2 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1505,9 +1505,9 @@ class PipelineExecutor: "table") else None ) - # Prefer an explicit provider hint from table metadata when available. + # Prefer an explicit plugin hint from table metadata when available. # This keeps @N selectors working even when row payloads don't carry a - # provider key (or when they carry a table-type like tidal.album). + # plugin key (or when they carry a table-type like tidal.album). try: meta = ( current_table.get_table_metadata() @@ -1517,56 +1517,58 @@ class PipelineExecutor: except Exception: meta = None if isinstance(meta, dict): + _add(meta.get("plugin")) _add(meta.get("provider")) except Exception: logger.exception("Failed to inspect current_table/table metadata in _maybe_run_class_selector") for item in selected_items or []: if isinstance(item, dict): + _add(item.get("plugin")) _add(item.get("provider")) _add(item.get("store")) _add(item.get("table")) else: + _add(getattr(item, "plugin", None)) _add(getattr(item, "provider", None)) _add(getattr(item, "store", None)) _add(getattr(item, "table", None)) try: - from ProviderCore.registry import get_provider, is_known_provider_name + from ProviderCore.registry import get_plugin, is_known_plugin_name except Exception: - get_provider = None # type: ignore - is_known_provider_name = None # type: ignore + get_plugin = None # type: ignore + is_known_plugin_name = None # type: ignore - # If we have a table-type like "tidal.album", also try its provider prefix ("tidal") - # when that prefix is a registered provider name. - if is_known_provider_name is not None: + # If we have a table-type like "tidal.album", also try its plugin prefix ("tidal") + # when that prefix is a registered plugin name. + if is_known_plugin_name is not None: try: for key in list(candidates): if not isinstance(key, str): continue if "." not in key: continue - if is_known_provider_name(key): + if is_known_plugin_name(key): continue prefix = str(key).split(".", 1)[0].strip().lower() - if prefix and is_known_provider_name(prefix): + if prefix and is_known_plugin_name(prefix): _add(prefix) except Exception: - logger.exception("Failed while computing provider prefix heuristics in _maybe_run_class_selector") + logger.exception("Failed while computing plugin prefix heuristics in _maybe_run_class_selector") - if get_provider is not None: + if get_plugin is not None: for key in candidates: try: - if is_known_provider_name is not None and ( - not is_known_provider_name(key)): + if is_known_plugin_name is not None and ( + not is_known_plugin_name(key)): continue except Exception: - # If the predicate fails for any reason, fall back to legacy behavior. - logger.exception("is_known_provider_name predicate failed for key %s; falling back", key) + logger.exception("is_known_plugin_name predicate failed for key %s; falling back", key) try: - provider = get_provider(key, config) + provider = get_plugin(key, config) except Exception as exc: - logger.exception("Failed to load provider '%s' during selector resolution: %s", key, exc) + logger.exception("Failed to load plugin '%s' during selector resolution: %s", key, exc) continue selector = getattr(provider, "selector", None) if selector is None: @@ -1583,6 +1585,92 @@ class PipelineExecutor: if handled: return True + @staticmethod + def _maybe_expand_plugin_selection( + selected_items: List[Any], + *, + ctx: Any, + config: Dict[str, Any], + stage_table: Any, + ) -> Optional[List[Any]]: + candidates: list[str] = [] + + def _add(value: Any) -> None: + text = str(value or "").strip().lower() + if text and text not in candidates: + candidates.append(text) + + table_type = None + try: + table_type = stage_table.table if stage_table is not None and hasattr(stage_table, "table") else None + except Exception: + table_type = None + _add(table_type) + + try: + meta = ( + stage_table.get_table_metadata() + if stage_table is not None and hasattr(stage_table, "get_table_metadata") + else getattr(stage_table, "table_metadata", None) + ) + except Exception: + meta = None + if isinstance(meta, dict): + _add(meta.get("plugin")) + _add(meta.get("provider")) + + for item in selected_items or []: + if isinstance(item, dict): + _add(item.get("plugin")) + _add(item.get("provider")) + _add(item.get("table")) + _add(item.get("source")) + else: + _add(getattr(item, "plugin", None)) + _add(getattr(item, "provider", None)) + _add(getattr(item, "table", None)) + _add(getattr(item, "source", None)) + + try: + from ProviderCore.registry import get_plugin, is_known_plugin_name + except Exception: + return None + + for key in list(candidates): + if "." in key: + prefix = str(key).split(".", 1)[0].strip().lower() + if prefix and prefix not in candidates: + candidates.append(prefix) + + for key in candidates: + try: + if not is_known_plugin_name(key): + continue + except Exception: + continue + try: + plugin = get_plugin(key, config) + except Exception: + continue + if plugin is None: + continue + expand = getattr(plugin, "expand_selection", None) + if not callable(expand): + continue + try: + expanded = expand( + selected_items, + ctx=ctx, + stage_is_last=False, + table_type=str(table_type or ""), + ) + except Exception: + logger.exception("%s expand_selection failed", key) + return None + if expanded: + return list(expanded) + return None + store_keys: list[str] = [] for item in selected_items or []: if isinstance(item, dict): @@ -1998,10 +2086,10 @@ class PipelineExecutor: # IMPORTANT: Put selected row args *before* source_args. # Rationale: The cmdlet argument parser treats the *first* unknown # token as a positional value (e.g., URL). If `source_args` - # contain unknown flags (like -provider which download-file does + # contain unknown flags (like a removed legacy flag that download-file does # not declare), they could be misinterpreted as the positional # URL argument and cause attempts to download strings like - # "-provider" (which is invalid). By placing selection args + # not accept). By placing selection args # first we ensure the intended URL/selection token is parsed # as the positional URL and avoid this class of parsing errors. expanded_stage: List[str] = cmd_list + selected_row_args + source_args @@ -2081,66 +2169,15 @@ class PipelineExecutor: print("No items matched selection in pipeline\n") return False, None - # Provider selection expansion (non-terminal): allow certain provider tables - # (e.g. tidal.album) to expand to multiple downstream items when the user - # pipes into another stage (e.g. @N | .mpv or @N | add-file). - table_type_hint = None - try: - table_type_hint = ( - stage_table.table - if stage_table is not None and hasattr(stage_table, "table") - else None + if stages: + expanded = PipelineExecutor._maybe_expand_plugin_selection( + filtered, + ctx=ctx, + config=config, + stage_table=stage_table, ) - except Exception: - table_type_hint = None - - if stages and isinstance(table_type_hint, str) and table_type_hint.strip().lower() == "tidal.album": - try: - from ProviderCore.registry import get_provider - - prov = get_provider("tidal", config) - except Exception: - prov = None - - if prov is not None and hasattr(prov, "_extract_album_selection_context") and hasattr(prov, "_tracks_for_album"): - try: - album_contexts = prov._extract_album_selection_context(filtered) # type: ignore[attr-defined] - except Exception: - album_contexts = [] - - track_items: List[Any] = [] - seen_track_ids: set[int] = set() - for album_id, album_title, artist_name in album_contexts or []: - try: - track_results = prov._tracks_for_album( # type: ignore[attr-defined] - album_id=album_id, - album_title=album_title, - artist_name=artist_name, - limit=500, - ) - except Exception: - track_results = [] - for tr in track_results or []: - try: - md = getattr(tr, "full_metadata", None) - tid = None - if isinstance(md, dict): - raw_id = md.get("trackId") or md.get("id") - try: - tid = int(raw_id) if raw_id is not None else None - except Exception: - tid = None - if tid is not None: - if tid in seen_track_ids: - continue - seen_track_ids.add(tid) - except Exception: - logger.exception("Failed to extract/parse track metadata in album processing") - track_items.append(tr) - - if track_items: - filtered = track_items - table_type_hint = "tidal.track" + if expanded: + filtered = expanded if PipelineExecutor._maybe_run_class_selector( ctx, @@ -2177,6 +2214,16 @@ class PipelineExecutor: except Exception: logger.exception("Failed to determine current_table for selection auto-insert; defaulting to None") current_table = None + table_type_hint = None + try: + raw_table_type = ( + stage_table.table + if stage_table is not None and hasattr(stage_table, "table") else None + ) + if isinstance(raw_table_type, str) and raw_table_type.strip(): + table_type_hint = raw_table_type + except Exception: + table_type_hint = None table_type = None try: if isinstance(table_type_hint, str) and table_type_hint.strip(): diff --git a/SYS/result_table_adapters.py b/SYS/result_table_adapters.py index ea23df7..2ce17df 100644 --- a/SYS/result_table_adapters.py +++ b/SYS/result_table_adapters.py @@ -1,10 +1,4 @@ -"""Provider registry for ResultTable API (breaking, strict API). - -Providers register themselves here with an adapter and optional column factory -and selection function. Consumers (cmdlets) can look up providers by name and -obtain the columns and selection behavior for building tables and for selection -args used by subsequent cmdlets. -""" +"""Plugin registry for the strict ResultTable API.""" from __future__ import annotations from dataclasses import dataclass @@ -18,7 +12,7 @@ SelectionFn = Callable[[ResultModel], List[str]] @dataclass -class Provider: +class Plugin: name: str adapter: ProviderAdapter # columns can be a static list or a factory that derives columns from sample rows @@ -28,7 +22,7 @@ class Provider: def get_columns(self, rows: Optional[Iterable[ResultModel]] = None) -> List[ColumnSpec]: if self.columns is None: - raise ValueError(f"provider '{self.name}' must define columns") + raise ValueError(f"plugin '{self.name}' must define columns") if callable(self.columns): rows_list = list(rows) if rows is not None else [] @@ -37,13 +31,13 @@ class Provider: cols = list(self.columns) if not cols: - raise ValueError(f"provider '{self.name}' produced no columns") + raise ValueError(f"plugin '{self.name}' produced no columns") return cols def selection_args(self, row: ResultModel) -> List[str]: if not callable(self.selection_fn): - raise ValueError(f"provider '{self.name}' must define a selection function") + raise ValueError(f"plugin '{self.name}' must define a selection function") sel = list(self.selection_fn(ensure_result_model(row))) return sel @@ -54,7 +48,7 @@ class Provider: try: rows = [ensure_result_model(r) for r in self.adapter(items)] except Exception as exc: - raise RuntimeError(f"provider '{self.name}' adapter failed") from exc + raise RuntimeError(f"plugin '{self.name}' adapter failed") from exc cols = self.get_columns(rows) return ResultTable(provider=self.name, rows=rows, columns=cols, meta=self.metadata or {}) @@ -82,37 +76,37 @@ class Provider: return [self.serialize_row(r) for r in rows] -_PROVIDERS: Dict[str, Provider] = {} +_PLUGINS: Dict[str, Plugin] = {} -def register_provider( +def register_plugin( name: str, adapter: ProviderAdapter, *, columns: Union[List[ColumnSpec], ColumnFactory], selection_fn: SelectionFn, metadata: Optional[Dict[str, Any]] = None, -) -> Provider: +) -> Plugin: name = str(name or "").strip().lower() if not name: - raise ValueError("provider name required") - if name in _PROVIDERS: - raise ValueError(f"provider already registered: {name}") + raise ValueError("plugin name required") + if name in _PLUGINS: + raise ValueError(f"plugin already registered: {name}") if columns is None: - raise ValueError("provider registration requires columns") + raise ValueError("plugin registration requires columns") if selection_fn is None: - raise ValueError("provider registration requires selection_fn") - p = Provider(name=name, adapter=adapter, columns=columns, selection_fn=selection_fn, metadata=metadata) - _PROVIDERS[name] = p - return p + raise ValueError("plugin registration requires selection_fn") + plugin = Plugin(name=name, adapter=adapter, columns=columns, selection_fn=selection_fn, metadata=metadata) + _PLUGINS[name] = plugin + return plugin -def get_provider(name: str) -> Provider: +def get_plugin(name: str) -> Plugin: normalized = str(name or "").lower() - if normalized not in _PROVIDERS: - raise KeyError(f"provider not registered: {name}") - return _PROVIDERS[normalized] + if normalized not in _PLUGINS: + raise KeyError(f"plugin not registered: {name}") + return _PLUGINS[normalized] -def list_providers() -> List[str]: - return list(_PROVIDERS.keys()) +def list_plugins() -> List[str]: + return list(_PLUGINS.keys()) diff --git a/SYS/rich_display.py b/SYS/rich_display.py index 62379f0..f7be249 100644 --- a/SYS/rich_display.py +++ b/SYS/rich_display.py @@ -148,7 +148,7 @@ def show_store_config_panel( def show_available_providers_panel(provider_names: List[str]) -> None: - """Show a Rich panel listing available/configured providers.""" + """Show a Rich panel listing available/configured plugins.""" from rich.columns import Columns from rich.console import Group @@ -164,13 +164,13 @@ def show_available_providers_panel(provider_names: List[str]) -> None: ) group = Group( - Text("The following providers are configured and ready to use:\n"), + Text("The following plugins are configured and ready to use:\n"), cols ) panel = Panel( group, - title="[bold green]Configured Providers[/bold green]", + title="[bold green]Configured Plugins[/bold green]", border_style="green", padding=(1, 2) ) diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index dd8e29f..2646fcd 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -526,9 +526,6 @@ class HydrusNetwork(Store): # Upload file if not already present if not file_exists: - debug( - f"{self._log_prefix()} Uploading: {file_path.name}" - ) response = client.add_file(file_path) # Extract hash from response @@ -553,8 +550,15 @@ class HydrusNetwork(Store): hydrus_hash = None if not hydrus_hash or len(str(hydrus_hash)) != 64: - debug( - f"{self._log_prefix()} Hydrus response hash missing/invalid; using precomputed hash" + debug_panel( + "Hydrus upload fallback", + [ + ("store", self.NAME), + ("file", file_path.name), + ("reason", "response hash missing/invalid"), + ("fallback_hash", file_hash), + ], + border_style="yellow", ) hydrus_hash = file_hash diff --git a/TUI.py b/TUI.py index 50dccd1..e3cb482 100644 --- a/TUI.py +++ b/TUI.py @@ -613,14 +613,14 @@ class PipelineHubApp(App): # Run startup check automatically self._run_pipeline_background(".status") - # Provide a visible startup summary of configured providers/stores for debugging + # Provide a visible startup summary of configured plugins/stores for debugging try: cfg = load_config() or {} provs = list(cfg.get("provider", {}).keys()) if isinstance(cfg.get("provider"), dict) else [] stores = list(cfg.get("store", {}).keys()) if isinstance(cfg.get("store"), dict) else [] prov_display = ", ".join(provs[:10]) + ("..." if len(provs) > 10 else "") store_display = ", ".join(stores[:10]) + ("..." if len(stores) > 10 else "") - self._append_log_line(f"Startup config: providers={len(provs)} ({prov_display or '(none)'}), stores={len(stores)} ({store_display or '(none)'}), db={db.db_path.name}") + self._append_log_line(f"Startup config: plugins={len(provs)} ({prov_display or '(none)'}), stores={len(stores)} ({store_display or '(none)'}), db={db.db_path.name}") except Exception: logger.exception("Failed to produce startup config summary") diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index 4ece4f3..c04de87 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -23,7 +23,7 @@ from SYS.config import ( from SYS.database import db from SYS.logger import log, debug from Store.registry import _discover_store_classes, _required_keys_for -from ProviderCore.registry import list_providers +from ProviderCore.registry import get_plugin, list_plugins from TUI.modalscreen.matrix_room_picker import MatrixRoomPicker from TUI.modalscreen.selection_modal import SelectionModal import logging @@ -177,7 +177,7 @@ class ConfigModal(ModalScreen): with ListView(id="category-list"): yield ListItem(Label("Global Settings"), id="cat-globals") yield ListItem(Label("Stores"), id="cat-stores") - yield ListItem(Label("Providers"), id="cat-providers") + yield ListItem(Label("Plugins"), id="cat-providers") yield ListItem(Label("Tools"), id="cat-tools") with Vertical(id="config-content"): @@ -187,7 +187,7 @@ class ConfigModal(ModalScreen): # Durable synchronous save: waits and verifies DB persisted critical keys yield Button("Save (durable)", variant="primary", id="save-durable-btn") yield Button("Add Store", variant="primary", id="add-store-btn") - yield Button("Add Provider", variant="primary", id="add-provider-btn") + yield Button("Add Plugin", variant="primary", id="add-provider-btn") yield Button("Add Tool", variant="primary", id="add-tool-btn") yield Button("Back", id="back-btn") yield Button("Close", variant="error", id="cancel-btn") @@ -381,10 +381,10 @@ class ConfigModal(ModalScreen): container.mount(row) def render_providers(self, container: ScrollableContainer) -> None: - container.mount(Label("Configured Providers", classes="config-label")) + container.mount(Label("Configured Plugins", classes="config-label")) providers = self.config_data.get("provider", {}) if not providers: - container.mount(Static("No providers configured.")) + container.mount(Static("No plugins configured.")) else: for i, (name, _) in enumerate(providers.items()): edit_id = f"edit-provider-{i}" @@ -448,9 +448,9 @@ class ConfigModal(ModalScreen): # Fetch Provider schema if item_type == "provider": - from ProviderCore.registry import get_provider_class + from ProviderCore.registry import get_plugin_class try: - pcls = get_provider_class(item_name) + pcls = get_plugin_class(item_name) if pcls and hasattr(pcls, "config_schema") and callable(pcls.config_schema): for field_def in pcls.config_schema(): k = field_def.get("key") @@ -625,9 +625,9 @@ class ConfigModal(ModalScreen): # If it's a provider, we might have required keys (legacy check fallback) if item_type == "provider": # 2. Legacy required_config_keys - from ProviderCore.registry import get_provider_class + from ProviderCore.registry import get_plugin_class try: - pcls = get_provider_class(item_name) + pcls = get_plugin_class(item_name) if pcls: required_keys = pcls.required_config_keys() for rk in required_keys: @@ -886,18 +886,18 @@ class ConfigModal(ModalScreen): logger.exception("Failed to inspect store class config_schema for '%s'", stype) self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected) elif bid == "add-provider-btn": - provider_names = list(list_providers().keys()) + provider_names = list(list_plugins().keys()) options = [] - from ProviderCore.registry import get_provider_class + from ProviderCore.registry import get_plugin_class for ptype in provider_names: - pcls = get_provider_class(ptype) + pcls = get_plugin_class(ptype) if pcls and hasattr(pcls, "config_schema") and callable(pcls.config_schema): try: if pcls.config_schema(): options.append(ptype) except Exception: logger.exception("Failed to inspect provider class config_schema for '%s'", ptype) - self.app.push_screen(SelectionModal("Select Provider Type", options), callback=self.on_provider_type_selected) + self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected) elif bid == "add-tool-btn": # Discover tool modules that advertise a config_schema() options = [] @@ -1067,9 +1067,9 @@ class ConfigModal(ModalScreen): # For providers, they are usually top-level entries in 'provider' dict if ptype not in self.config_data["provider"]: - from ProviderCore.registry import get_provider_class + from ProviderCore.registry import get_plugin_class try: - pcls = get_provider_class(ptype) + pcls = get_plugin_class(ptype) new_config = {} if pcls: # Use schema for defaults @@ -1273,9 +1273,9 @@ class ConfigModal(ModalScreen): @work(thread=True) def _matrix_test_background(self) -> None: try: - from Provider.matrix import Matrix - - provider = Matrix(self.config_data) + provider = get_plugin("matrix", self.config_data) + if provider is None: + raise RuntimeError("Matrix plugin unavailable") rooms = provider.list_rooms() self.app.call_from_thread(self._matrix_test_result, True, rooms, None) except Exception as exc: @@ -1433,9 +1433,9 @@ class ConfigModal(ModalScreen): @work(thread=True) def _matrix_load_background(self) -> None: try: - from Provider.matrix import Matrix - - provider = Matrix(self.config_data) + provider = get_plugin("matrix", self.config_data) + if provider is None: + raise RuntimeError("Matrix plugin unavailable") rooms = provider.list_rooms() self.app.call_from_thread(self._matrix_load_result, True, rooms, None) except Exception as exc: @@ -1626,8 +1626,9 @@ class ConfigModal(ModalScreen): return [] try: - from Provider.matrix import Matrix - provider = Matrix(self.config_data) + provider = get_plugin("matrix", self.config_data) + if provider is None: + return [] rooms = provider.list_rooms(room_ids=ids_list) return rooms or [] except Exception as exc: @@ -1870,9 +1871,9 @@ class ConfigModal(ModalScreen): required_keys = list(_required_keys_for(classes[stype])) section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {}) elif item_type == "provider": - from ProviderCore.registry import get_provider_class + from ProviderCore.registry import get_plugin_class try: - pcls = get_provider_class(item_name) + pcls = get_plugin_class(item_name) if pcls: # Collect required keys from schema if hasattr(pcls, "config_schema") and callable(pcls.config_schema): diff --git a/TUI/modalscreen/matrix_room_picker.py b/TUI/modalscreen/matrix_room_picker.py index d8542e6..fc6a80b 100644 --- a/TUI/modalscreen/matrix_room_picker.py +++ b/TUI/modalscreen/matrix_room_picker.py @@ -8,6 +8,7 @@ from textual.screen import ModalScreen from textual.widgets import Static, Button, Checkbox, ListView, ListItem from textual import work from rich.text import Text +from ProviderCore.registry import get_plugin import logging logger = logging.getLogger(__name__) @@ -181,8 +182,9 @@ class MatrixRoomPicker(ModalScreen[List[str]]): @work(thread=True) def _load_rooms_background(self) -> None: try: - from Provider.matrix import Matrix - provider = Matrix(self.config) + provider = get_plugin("matrix", self.config) + if provider is None: + raise RuntimeError("Matrix plugin unavailable") rooms = provider.list_rooms() self.app.call_from_thread(self._apply_room_results, rooms, None) except Exception as exc: diff --git a/TUI/modalscreen/search.py b/TUI/modalscreen/search.py index 770033a..ccd9c1e 100644 --- a/TUI/modalscreen/search.py +++ b/TUI/modalscreen/search.py @@ -16,7 +16,7 @@ import asyncio sys.path.insert(0, str(Path(__file__).parent.parent)) from SYS.config import load_config, resolve_output_dir from SYS.result_table import Table -from ProviderCore.registry import get_search_provider +from ProviderCore.registry import get_search_plugin logger = logging.getLogger(__name__) @@ -174,7 +174,7 @@ class SearchModal(ModalScreen): self.current_worker.log_step(f"Connecting to {source}...") try: - provider = get_search_provider(source) + provider = get_search_plugin(source) if not provider: logger.error(f"[search-modal] Provider not available: {source}") if self.current_worker: @@ -380,7 +380,7 @@ class SearchModal(ModalScreen): config = load_config() output_dir = resolve_output_dir(config) - provider = get_search_provider("openlibrary", config=config) + provider = get_search_plugin("openlibrary", config=config) if not provider: logger.error("[search-modal] Provider not available: openlibrary") return diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 901910c..bb02fed 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -199,10 +199,10 @@ class SharedArgs: type="string", description="http parser", ) - PROVIDER = CmdletArg( - name="provider", + PLUGIN = CmdletArg( + name="plugin", type="string", - description="selects provider", + description="selects plugin", ) @staticmethod @@ -538,10 +538,13 @@ class Cmdlet: elif low in flags.get('tag', set()): # handle tag """ - return { - arg.name: self.get_flags(arg.name) - for arg in self.arg - } + registry: Dict[str, set[str]] = {} + for arg in self.arg: + try: + registry[arg.name] = {str(flag).lower() for flag in arg.to_flags()} + except Exception: + registry[arg.name] = {flag.lower() for flag in self.get_flags(arg.name)} + return registry # Tag groups cache (loaded from JSON config file) @@ -642,10 +645,14 @@ def parse_cmdlet_args(args: Sequence[str], else: flagged_args.append(spec) - # Register all prefix variants for flagged lookup - arg_spec_map[canonical_name.lower()] = canonical_name # bare name - arg_spec_map[f"-{canonical_name}".lower()] = canonical_name # single dash - arg_spec_map[f"--{canonical_name}".lower()] = canonical_name # double dash + # Register all supported flag variants, including legacy aliases. + arg_spec_map[canonical_name.lower()] = canonical_name # bare canonical name + try: + for flag in spec.to_flags(): + arg_spec_map[str(flag).lower()] = canonical_name + except Exception: + arg_spec_map[f"-{canonical_name}".lower()] = canonical_name + arg_spec_map[f"--{canonical_name}".lower()] = canonical_name # Parse arguments i = 0 @@ -3143,16 +3150,6 @@ def register_url_with_local_library( """ # Folder store removed; local library URL registration is disabled. return False - - -try: - # Provider-specific implementation lives with the provider code. - from Provider.tidal_manifest import resolve_tidal_manifest_path -except Exception: # pragma: no cover - def resolve_tidal_manifest_path(item: Any) -> Optional[str]: - _ = item - return None - def check_url_exists_in_storage( urls: Sequence[str], storage: Any, diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 3120543..9cca9bd 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -176,14 +176,14 @@ class Add_File(Cmdlet): super().__init__( name="add-file", summary= - "Ingest a local media file to a store backend, file provider, or local directory.", + "Ingest a local media file to a store backend, upload plugin, or local directory.", usage= - "add-file (-path | ) (-storage | -provider ) [-delete]", + "add-file (-path | ) (-storage | -plugin ) [-delete]", arg=[ SharedArgs.PATH, SharedArgs.STORE, SharedArgs.URL, - SharedArgs.PROVIDER, + SharedArgs.PLUGIN, CmdletArg( name="delete", type="flag", @@ -198,7 +198,7 @@ class Add_File(Cmdlet): " hydrus: Upload to Hydrus database with metadata tagging", " local: Copy file to local directory", " : Copy file to specified directory", - "- File provider options (use -provider):", + "- Upload plugin options (use -plugin):", " 0x0: Upload to 0x0.st for temporary hosting", " file.io: Upload to file.io for temporary hosting", " internetarchive: Upload to archive.org (optional tag: ia: to upload into an existing item)", @@ -224,13 +224,13 @@ class Add_File(Cmdlet): path_arg = parsed.get("path") location = parsed.get("store") source_url_arg = parsed.get("url") - provider_name = parsed.get("provider") + plugin_name = parsed.get("plugin") delete_after = parsed.get("delete", False) # Convenience: when piping a file into add-file, allow `-path ` # to act as the destination export directory. # Example: screen-shot "https://..." | add-file -path "C:\Users\Admin\Desktop" - if path_arg and not location and not provider_name: + if path_arg and not location and not plugin_name: try: candidate_dir = Path(str(path_arg)) if candidate_dir.exists() and candidate_dir.is_dir(): @@ -263,7 +263,7 @@ class Add_File(Cmdlet): dir_scan_results: Optional[List[Dict[str, Any]]] = None explicit_path_list_results: Optional[List[Dict[str, Any]]] = None - if path_arg and location and not provider_name: + if path_arg and location and not plugin_name: # Support comma-separated path lists: -path "file1,file2,file3" # This is the mechanism used by @N expansion for directory tables. try: @@ -403,7 +403,7 @@ class Add_File(Cmdlet): ("result_type", type(result).__name__), ("items", total_items), ("location", location), - ("provider", provider_name), + ("plugin", plugin_name), ("delete", delete_after), ], border_style="cyan", @@ -599,8 +599,8 @@ class Add_File(Cmdlet): export_destination=(Path(location) if location and not is_storage_backend_location else None), store_instance=storage_registry, ) - if not media_path and provider_name: - media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source( + if not media_path and plugin_name: + media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source( pipe_obj, config, storage_registry ) if media_path: @@ -610,7 +610,7 @@ class Add_File(Cmdlet): [ ("path", media_path), ("hash", file_hash or "N/A"), - ("provider", provider_name or "local"), + ("plugin", plugin_name or "local"), ], border_style="green", ) @@ -635,10 +635,10 @@ class Add_File(Cmdlet): progress.step("hashing file") progress.step("ingesting file") - if provider_name: - code = self._handle_provider_upload( + if plugin_name: + code = self._handle_plugin_upload( media_path, - provider_name, + plugin_name, pipe_obj, config, delete_after_item @@ -1365,7 +1365,7 @@ class Add_File(Cmdlet): hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None) return candidate, hash_hint, None - downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_provider_result( + downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_plugin_result( result, pipe_obj, config, @@ -1393,45 +1393,41 @@ class Add_File(Cmdlet): return normalized @staticmethod - def _maybe_download_provider_result( + def _maybe_download_plugin_result( result: Any, pipe_obj: models.PipeObject, config: Dict[str, Any], ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: - provider_key = None + plugin_key = None for source in ( pipe_obj.provider, + get_field(result, "plugin"), get_field(result, "provider"), get_field(result, "table"), ): candidate = Add_File._normalize_provider_key(source) if candidate: - provider_key = candidate + plugin_key = candidate break - if not provider_key: + if not plugin_key: return None, None, None - provider = get_search_provider(provider_key, config) - if provider is None: + from ProviderCore.registry import get_search_plugin + + plugin = get_search_plugin(plugin_key, config) + if plugin is None: return None, None, None - # Check for specialized download helper (used by AllDebrid and potentially others) - handler = getattr(provider, "download_for_pipe_result", None) - if not callable(handler): - # Fallback: check class if it's a classmethod and instance didn't have it (unlikely but safe) - handler = getattr(type(provider), "download_for_pipe_result", None) - - if callable(handler): - try: - return handler(result, pipe_obj, config) - except Exception as exc: - debug(f"[add-file] Provider '{provider_key}' download helper failed: {exc}") + try: + return plugin.resolve_pipe_result_download(result, pipe_obj) + except Exception as exc: + debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}") return None, None, None @staticmethod - def _download_provider_source( + def _download_piped_source( pipe_obj: models.PipeObject, config: Dict[str, Any], store_instance: Optional[Any], @@ -2152,23 +2148,23 @@ class Add_File(Cmdlet): return 0 @staticmethod - def _handle_provider_upload( + def _handle_plugin_upload( media_path: Path, - provider_name: str, + plugin_name: str, pipe_obj: models.PipeObject, config: Dict[str, Any], delete_after: bool, ) -> int: - """Handle uploading to a file provider (e.g. 0x0).""" - from ProviderCore.registry import get_file_provider + """Handle uploading via an upload plugin (e.g. 0x0).""" + from ProviderCore.registry import get_upload_plugin - log(f"Uploading via {provider_name}: {media_path.name}", file=sys.stderr) + log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr) try: - file_provider = get_file_provider(provider_name, config) + file_provider = get_upload_plugin(plugin_name, config) if not file_provider: - log(f"File provider '{provider_name}' not available", file=sys.stderr) + log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr) return 1 hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj) @@ -2183,8 +2179,8 @@ class Add_File(Cmdlet): # Update PipeObject and emit extra_updates: Dict[str, Any] = { - "provider": provider_name, - "provider_url": hoster_url, + "plugin": plugin_name, + "plugin_url": hoster_url, } if isinstance(pipe_obj.extra, dict): # Also track hoster URL as a url for downstream steps @@ -2197,7 +2193,7 @@ class Add_File(Cmdlet): Add_File._update_pipe_object_destination( pipe_obj, hash_value=f_hash or "unknown", - store=provider_name or "provider", + store=plugin_name or "plugin", path=file_path, tag=pipe_obj.tag, title=pipe_obj.title or (media_path.name if media_path else None), @@ -2445,9 +2441,6 @@ class Add_File(Cmdlet): try: adder = getattr(backend, "add_tag", None) if callable(adder): - debug( - f"[add-file] Applying {len(tags)} tag(s) post-upload to {backend_name}" - ) adder(resolved_hash, list(tags)) except Exception as exc: log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr) @@ -2479,48 +2472,72 @@ class Add_File(Cmdlet): try: setter = getattr(backend, "set_note", None) if callable(setter): - debug( - f"[add-file] Writing sub note (len={len(str(sub_note))}) to {backend_name}:{resolved_hash}" - ) setter(resolved_hash, "sub", sub_note) except Exception as exc: - debug(f"[add-file] sub note write failed: {exc}") + debug_panel( + "add-file note write failed", + [ + ("store", backend_name), + ("hash", resolved_hash), + ("note", "sub"), + ("error", exc), + ], + border_style="yellow", + ) lyric_note = Add_File._get_note_text(result, pipe_obj, "lyric") if lyric_note: try: setter = getattr(backend, "set_note", None) if callable(setter): - debug( - f"[add-file] Writing lyric note (len={len(str(lyric_note))}) to {backend_name}:{resolved_hash}" - ) setter(resolved_hash, "lyric", lyric_note) except Exception as exc: - debug(f"[add-file] lyric note write failed: {exc}") + debug_panel( + "add-file note write failed", + [ + ("store", backend_name), + ("hash", resolved_hash), + ("note", "lyric"), + ("error", exc), + ], + border_style="yellow", + ) chapters_note = Add_File._get_note_text(result, pipe_obj, "chapters") if chapters_note: try: setter = getattr(backend, "set_note", None) if callable(setter): - debug( - f"[add-file] Writing chapters note (len={len(str(chapters_note))}) to {backend_name}:{resolved_hash}" - ) setter(resolved_hash, "chapters", chapters_note) except Exception as exc: - debug(f"[add-file] chapters note write failed: {exc}") + debug_panel( + "add-file note write failed", + [ + ("store", backend_name), + ("hash", resolved_hash), + ("note", "chapters"), + ("error", exc), + ], + border_style="yellow", + ) caption_note = Add_File._get_note_text(result, pipe_obj, "caption") if caption_note: try: setter = getattr(backend, "set_note", None) if callable(setter): - debug( - f"[add-file] Writing caption note (len={len(str(caption_note))}) to {backend_name}:{resolved_hash}" - ) setter(resolved_hash, "caption", caption_note) except Exception as exc: - debug(f"[add-file] caption note write failed: {exc}") + debug_panel( + "add-file note write failed", + [ + ("store", backend_name), + ("hash", resolved_hash), + ("note", "caption"), + ("error", exc), + ], + border_style="yellow", + ) meta: Dict[str, Any] = {} diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index afea22e..2a4fab8 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -2,7 +2,7 @@ Supports: - Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp) -- Piped provider items (uses provider.download when available) +- Piped plugin items (uses plugin.download when available) - Streaming sites via yt-dlp (YouTube, Bandcamp, etc.) """ @@ -18,7 +18,7 @@ from contextlib import AbstractContextManager, nullcontext from API.HTTP import _download_direct_file from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult -from SYS.logger import log, debug, debug_panel, is_debug_enabled +from SYS.logger import log, debug_panel, is_debug_enabled from SYS.payload_builders import build_file_result_payload, build_table_result_payload from SYS.pipeline_progress import PipelineProgress from SYS.result_table import Table @@ -32,20 +32,6 @@ from SYS.selection_builder import ( ) from SYS.utils import sha256_file -from tool.ytdlp import ( - YtDlpTool, - _best_subtitle_sidecar, - _SUBTITLE_EXTS, - _download_with_timeout, - _format_chapters_note, - _read_text_file, - is_url_supported_by_ytdlp, - is_browseable_format, - format_for_table_selection, - list_formats, - probe_url, -) - from . import _shared as sh Cmdlet = sh.Cmdlet @@ -60,12 +46,6 @@ resolve_target_dir = sh.resolve_target_dir coerce_to_path = sh.coerce_to_path build_pipeline_preview = sh.build_pipeline_preview -# URI scheme prefixes owned by AllDebrid (magic-link and emoji shorthand). -# Defined once here so every method in this file references the same constant. -_ALLDEBRID_PREFIXES: tuple[str, ...] = ("alldebrid:", "alldebrid🧲") -_FORMAT_INDEX_RE = re.compile(r"^\s*#?\d+\s*$") - - class Download_File(Cmdlet): """Class-based download-file cmdlet - direct HTTP downloads.""" @@ -80,7 +60,7 @@ class Download_File(Cmdlet): "download-http"], arg=[ SharedArgs.URL, - SharedArgs.PROVIDER, + SharedArgs.PLUGIN, SharedArgs.PATH, SharedArgs.QUERY, QueryArg( @@ -123,7 +103,7 @@ class Download_File(Cmdlet): border_style="cyan", ) except Exception: - debug(f"[download-file] run invoked with args: {list(args)}") + pass return self._run_impl(result, args, config) @staticmethod @@ -135,11 +115,15 @@ class Download_File(Cmdlet): return resolved @staticmethod - def _selection_run_label(run_args: Sequence[str]) -> str: + def _selection_run_label( + run_args: Sequence[str], + *, + extra_url_prefixes: Sequence[str] = (), + ) -> str: try: urls = extract_urls_from_selection_args( run_args, - extra_url_prefixes=_ALLDEBRID_PREFIXES, + extra_url_prefixes=extra_url_prefixes, ) if urls: return str(urls[0]) @@ -176,6 +160,87 @@ class Download_File(Cmdlet): return True, total, index, label + @staticmethod + def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]: + loader = registry.get("list_selection_url_prefixes") + if not callable(loader): + return [] + try: + values = loader() or [] + except Exception: + return [] + return [str(value).strip().lower() for value in values if str(value or "").strip()] + + def _emit_plugin_items( + self, + *, + items: Sequence[Any], + config: Dict[str, Any], + ) -> int: + emitted = 0 + for item in items: + if not isinstance(item, dict): + continue + pipeline_context.emit(item) + if item.get("url"): + try: + pipe_obj = coerce_to_pipe_object(item) + register_url_with_local_library(pipe_obj, config) + except Exception: + pass + emitted += 1 + return emitted + + def _consume_plugin_download_result( + self, + *, + result: Any, + config: Dict[str, Any], + ) -> tuple[int, Optional[int], bool]: + if result is None: + return 0, None, False + + if isinstance(result, list): + if result and all(isinstance(item, dict) for item in result): + return self._emit_plugin_items(items=result, config=config), 0, True + return 0, None, False + + if not isinstance(result, dict): + return 0, None, False + + action = str( + result.get("action") + or result.get("provider_action") + or "" + ).strip().lower() + + if action in {"emit_items", "emit_pipe_objects"}: + items = result.get("items") or [] + exit_code = result.get("exit_code") + emitted = self._emit_plugin_items( + items=items if isinstance(items, list) else [], + config=config, + ) + try: + normalized_exit = int(exit_code) if exit_code is not None else 0 + except Exception: + normalized_exit = 0 + return emitted, normalized_exit, True + + if action == "handled": + exit_code = result.get("exit_code") + try: + normalized_exit = int(exit_code) if exit_code is not None else 0 + except Exception: + normalized_exit = 0 + try: + downloaded = int(result.get("downloaded") or 0) + except Exception: + downloaded = 0 + return downloaded, normalized_exit, True + + return 0, None, False + def _process_explicit_urls( self, *, @@ -187,6 +252,8 @@ class Download_File(Cmdlet): registry: Dict[str, Any], progress: PipelineProgress, + parsed: Dict[str, Any], + args: Sequence[str], context_items: Sequence[Any] = (), ) -> tuple[int, Optional[int]]: @@ -201,12 +268,11 @@ class Download_File(Cmdlet): pass SearchResult = registry.get("SearchResult") - get_provider = registry.get("get_provider") - match_provider_name_for_url = registry.get("match_provider_name_for_url") + get_plugin = registry.get("get_plugin") + match_plugin_name_for_url = registry.get("match_plugin_name_for_url") for idx, url in enumerate(raw_urls, 1): try: - debug(f"Processing URL: {url}") try: display_total = batch_total if batch_total > 0 else total_urls display_index = batch_index if batch_total > 0 else idx @@ -220,18 +286,17 @@ class Download_File(Cmdlet): # Check providers first provider_name = None - if match_provider_name_for_url: + if match_plugin_name_for_url: try: - provider_name = match_provider_name_for_url(str(url)) + provider_name = match_plugin_name_for_url(str(url)) except Exception: pass provider = None - if provider_name and get_provider: - provider = get_provider(provider_name, config) + if provider_name and get_plugin: + provider = get_plugin(provider_name, config) if provider: - debug(f"Provider {provider_name} claimed {url}") try: # Try generic handle_url handled = False @@ -260,7 +325,6 @@ class Download_File(Cmdlet): request_metadata.setdefault("magnet_id", magnet_id) if SearchResult is None: - debug("Provider download_items requested but SearchResult unavailable") continue sr = SearchResult( @@ -282,6 +346,17 @@ class Download_File(Cmdlet): downloaded_count += int(downloaded_extra) continue + plugin_downloaded, plugin_exit, plugin_handled = self._consume_plugin_download_result( + result=path, + config=config, + ) + if plugin_handled: + downloaded_count += plugin_downloaded + if plugin_exit is not None and plugin_downloaded == 0: + return downloaded_count, int(plugin_exit) + if plugin_downloaded: + continue + path_value = path.get("path") or path.get("file_path") extra_meta = path.get("metadata") or path.get("full_metadata") title_hint = path.get("title") or path.get("name") @@ -309,15 +384,45 @@ class Download_File(Cmdlet): provider_hint=provider_name ) downloaded_count += 1 - else: - debug(f"Provider {provider_name} handled URL without file output") continue except Exception as e: - debug(f"Provider {provider_name} handle_url error: {e}") + debug_panel( + "download-file provider error", + [ + ("plugin", provider_name), + ("url", url), + ("operation", "handle_url"), + ("error", e), + ], + border_style="yellow", + ) # Try generic download_url if not already handled if not handled and hasattr(provider, "download_url"): - res = provider.download_url(str(url), final_output_dir) + try: + res = provider.download_url( + str(url), + final_output_dir, + parsed=parsed, + args=list(args), + progress=progress, + quiet_mode=quiet_mode, + context_items=list(context_items or []), + ) + except TypeError: + res = provider.download_url(str(url), final_output_dir) + + plugin_downloaded, plugin_exit, plugin_handled = self._consume_plugin_download_result( + result=res, + config=config, + ) + if plugin_handled: + downloaded_count += plugin_downloaded + if plugin_exit is not None and plugin_downloaded == 0: + return downloaded_count, int(plugin_exit) + if plugin_downloaded: + continue + if res: # Standardize result: can be Path, tuple(Path, Info), or dict with "path" p_val = None @@ -354,7 +459,6 @@ class Download_File(Cmdlet): pass if not handled: - debug(f"Provider {provider_name} matched URL but failed to download. Skipping direct fallback to avoid landing pages.") continue # Direct Download Fallback @@ -377,7 +481,6 @@ class Download_File(Cmdlet): config=config, ) downloaded_count += 1 - debug("✓ Downloaded and emitted") except DownloadError as e: log(f"Download failed for {url}: {e}", file=sys.stderr) @@ -419,13 +522,13 @@ class Download_File(Cmdlet): config: Dict[str, Any], ) -> List[Any]: - get_search_provider = registry.get("get_search_provider") + get_search_plugin = registry.get("get_search_plugin") expanded_items: List[Any] = [] for item in piped_items: try: provider_key = self._provider_key_from_item(item) - provider = get_search_provider(provider_key, config) if provider_key and get_search_provider else None + provider = get_search_plugin(provider_key, config) if provider_key and get_search_plugin else None # Generic hook: If provider has expand_item(item), use it. if provider and hasattr(provider, "expand_item") and callable(provider.expand_item): @@ -435,7 +538,14 @@ class Download_File(Cmdlet): expanded_items.extend(sub_items) continue except Exception as e: - debug(f"Provider {provider_key} expand_item failed: {e}") + debug_panel( + "download-file expand_item failed", + [ + ("plugin", provider_key), + ("error", e), + ], + border_style="yellow", + ) expanded_items.append(item) except Exception: @@ -456,7 +566,7 @@ class Download_File(Cmdlet): ) -> tuple[int, int]: downloaded_count = 0 queued_magnet_submissions = 0 - get_search_provider = registry.get("get_search_provider") + get_search_plugin = registry.get("get_search_plugin") SearchResult = registry.get("SearchResult") expanded_items = self._expand_provider_items( @@ -467,7 +577,6 @@ class Download_File(Cmdlet): total_items = len(expanded_items) processed_items = 0 - debug(f"[download-file] Processing {total_items} piped item(s)...") try: if total_items: @@ -482,8 +591,6 @@ class Download_File(Cmdlet): title = get_field(item, "title") target = get_field(item, "path") or get_field(item, "url") - debug(f"[download-file] Item {idx}/{total_items}: {title or target or 'unnamed'}") - media_kind = get_field(item, "media_kind") tags_val = get_field(item, "tag") tags_list: Optional[List[str]] @@ -521,9 +628,9 @@ class Download_File(Cmdlet): provider_sr = None provider_obj = None provider_key = self._provider_key_from_item(item) - if provider_key and get_search_provider and SearchResult: + if provider_key and get_search_plugin and SearchResult: # Reuse helper to derive the provider key from table/provider/source hints. - provider_obj = get_search_provider(provider_key, config) + provider_obj = get_search_plugin(provider_key, config) if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): try: @@ -542,18 +649,14 @@ class Download_File(Cmdlet): full_metadata=full_metadata if isinstance(full_metadata, dict) else {}, ) - debug( - f"[download-file] Downloading provider item via {table}: {sr.title}" - ) - # Preserve provider structure when possible (AllDebrid folders -> subfolders). + # Preserve plugin-managed output structure when a plugin encodes nested paths. output_dir = final_output_dir # Generic: allow provider to strict output_dir? # Using default output_dir for now. downloaded_path = provider_obj.download(sr, output_dir) provider_sr = sr - debug(f"[download-file] Provider download result: {downloaded_path}") if downloaded_path is None: try: @@ -576,10 +679,6 @@ class Download_File(Cmdlet): # Fallback: if we have a direct HTTP URL and no provider successfully handled it if (downloaded_path is None and not attempted_provider_download and isinstance(target, str) and target.startswith("http")): - - debug( - f"[download-file] Provider item looks like direct URL, downloading: {target}" - ) suggested_name = str(title).strip() if title is not None else None result_obj = _download_direct_file( @@ -603,7 +702,6 @@ class Download_File(Cmdlet): try: sr_md = getattr(provider_sr, "full_metadata", None) if isinstance(sr_md, dict) and sr_md: - debug(f"[download-file] Syncing full_metadata from provider_sr (keys={list(sr_md.keys())})") full_metadata = sr_md except Exception: pass @@ -620,7 +718,6 @@ class Download_File(Cmdlet): try: sr_tags = getattr(provider_sr, "tag", None) if isinstance(sr_tags, (set, list)) and sr_tags: - debug(f"[download-file] Syncing tags_list from provider_sr (count={len(sr_tags)})") # Re-sync tags_list with the potentially enriched provider_sr.tag tags_list = sorted([str(t) for t in sr_tags if t]) except Exception: @@ -766,6 +863,7 @@ class Download_File(Cmdlet): "tag": tag, } if provider_hint: + payload["plugin"] = str(provider_hint) payload["provider"] = str(provider_hint) if full_metadata: payload["metadata"] = full_metadata @@ -844,1916 +942,27 @@ class Download_File(Cmdlet): @staticmethod def _load_provider_registry() -> Dict[str, Any]: - """Lightweight accessor for provider helpers without hard dependencies.""" + """Lightweight accessor for plugin helpers without hard dependencies.""" try: from ProviderCore import registry as provider_registry # type: ignore from ProviderCore.base import SearchResult # type: ignore return { - "get_provider": getattr(provider_registry, "get_provider", None), - "get_search_provider": getattr(provider_registry, "get_search_provider", None), - "match_provider_name_for_url": getattr(provider_registry, "match_provider_name_for_url", None), + "get_plugin": getattr(provider_registry, "get_plugin", None), + "get_search_plugin": getattr(provider_registry, "get_search_plugin", None), + "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None), + "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None), "SearchResult": SearchResult, } except Exception: return { - "get_provider": None, - "get_search_provider": None, - "match_provider_name_for_url": None, + "get_plugin": None, + "get_search_plugin": None, + "match_plugin_name_for_url": None, + "list_selection_url_prefixes": None, "SearchResult": None, } - # === Streaming helpers (yt-dlp) === - - @staticmethod - def _filter_supported_urls(raw_urls: Sequence[str]) -> tuple[List[str], List[str]]: - supported = [url for url in (raw_urls or []) if is_url_supported_by_ytdlp(url)] - unsupported = list(set(raw_urls or []) - set(supported or [])) - return supported, unsupported - - @staticmethod - def _match_provider_urls( - raw_urls: Sequence[str], - registry: Dict[str, Any], - ) -> Dict[str, str]: - matches: Dict[str, str] = {} - if not raw_urls: - return matches - - match_provider_name_for_url = registry.get("match_provider_name_for_url") - if not callable(match_provider_name_for_url): - return matches - - for url in raw_urls: - try: - url_str = str(url or "").strip() - except Exception: - continue - if not url_str: - continue - try: - provider_name = match_provider_name_for_url(url_str) - except Exception: - provider_name = None - if provider_name: - matches[url_str] = str(provider_name).strip().lower() - - return matches - - def _parse_query_keyed_spec(self, query_spec: Optional[str]) -> Dict[str, List[str]]: - if not query_spec: - return {} - try: - keyed = self._parse_keyed_csv_spec(str(query_spec), default_key="hash") - if not keyed: - return {} - - def _alias(src: str, dest: str) -> None: - try: - values = keyed.get(src) - except Exception: - values = None - if not values: - return - try: - keyed.setdefault(dest, []).extend(list(values)) - except Exception: - pass - try: - keyed.pop(src, None) - except Exception: - pass - - for src in ("range", "ranges", "section", "sections"): - _alias(src, "clip") - for src in ("fmt", "f"): - _alias(src, "format") - for src in ("aud", "a"): - _alias(src, "audio") - - return keyed - except Exception: - return {} - - @staticmethod - def _extract_hash_override(query_spec: Optional[str], query_keyed: Dict[str, List[str]]) -> Optional[str]: - try: - hash_values = query_keyed.get("hash", []) if isinstance(query_keyed, dict) else [] - hash_candidate = hash_values[-1] if hash_values else None - if hash_candidate: - return sh.parse_single_hash_query(f"hash:{hash_candidate}") - - try: - has_non_hash_keys = bool( - query_keyed - and isinstance(query_keyed, dict) - and any(k for k in query_keyed.keys() if str(k).strip().lower() != "hash") - ) - except Exception: - has_non_hash_keys = False - if has_non_hash_keys: - return None - return sh.parse_single_hash_query(str(query_spec)) if query_spec else None - except Exception: - return None - - def _parse_clip_ranges_and_apply_items( - self, - *, - clip_spec: Optional[str], - query_keyed: Dict[str, List[str]], - parsed: Dict[str, Any], - query_spec: Optional[str], - ) -> tuple[Optional[List[tuple[int, int]]], bool, List[str]]: - clip_ranges: Optional[List[tuple[int, int]]] = None - clip_values: List[str] = [] - item_values: List[str] = [] - - def _uniq(values: Sequence[str]) -> List[str]: - seen: set[str] = set() - out: List[str] = [] - for v in values: - key = str(v) - if key in seen: - continue - seen.add(key) - out.append(v) - return out - - if clip_spec: - keyed = self._parse_keyed_csv_spec(str(clip_spec), default_key="clip") - clip_values.extend(keyed.get("clip", []) or []) - item_values.extend(keyed.get("item", []) or []) - - if query_keyed: - clip_values.extend(query_keyed.get("clip", []) or []) - item_values.extend(query_keyed.get("item", []) or []) - - clip_values = _uniq(clip_values) - item_values = _uniq(item_values) - - if item_values and not parsed.get("item"): - parsed["item"] = ",".join([v for v in item_values if v]) - - if clip_values: - clip_ranges = self._parse_time_ranges(",".join([v for v in clip_values if v])) - if not clip_ranges: - bad_spec = clip_spec or query_spec - log(f"Invalid clip format: {bad_spec}", file=sys.stderr) - return None, True, clip_values - - return clip_ranges, False, clip_values - - @staticmethod - def _init_storage(config: Dict[str, Any]) -> tuple[Optional[Any], bool]: - # Cache storage object in config to avoid excessive DB initialization in loops - if isinstance(config, dict) and "_storage_cache" in config: - cached = config["_storage_cache"] - if isinstance(cached, tuple) and len(cached) == 2: - return cached # type: ignore - - storage = None - hydrus_available = True - try: - from Store import Store - from API.HydrusNetwork import is_hydrus_available - - storage = Store(config=config or {}, suppress_debug=True) - hydrus_available = bool(is_hydrus_available(config or {})) - - # If any Hydrus store backend was successfully initialized in the Store - # registry, consider Hydrus available even if the global probe failed. - try: - from Store.HydrusNetwork import HydrusNetwork as _HydrusStoreClass - for bn in storage.list_backends(): - try: - backend = storage[bn] - if isinstance(backend, _HydrusStoreClass): - hydrus_available = True - break - except Exception: - continue - except Exception: - pass - - if isinstance(config, dict): - config["_storage_cache"] = (storage, hydrus_available) - except Exception as e: - debug(f"[download-file] Storage initialization error: {e}") - storage = None - return storage, hydrus_available - - @staticmethod - def _cookiefile_str(ytdlp_tool: YtDlpTool) -> Optional[str]: - try: - cookie_path = ytdlp_tool.resolve_cookiefile() - if cookie_path is not None and cookie_path.is_file(): - return str(cookie_path) - except Exception: - pass - return None - - def _list_formats_cached( - self, - u: str, - *, - playlist_items_value: Optional[str], - formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], - ytdlp_tool: YtDlpTool, - ) -> Optional[List[Dict[str, Any]]]: - key = f"{u}||{playlist_items_value or ''}" - if key in formats_cache: - return formats_cache[key] - fmts = list_formats( - u, - no_playlist=False, - playlist_items=playlist_items_value, - cookiefile=self._cookiefile_str(ytdlp_tool), - ) - formats_cache[key] = fmts - return fmts - - def _is_browseable_format(self, fmt: Any) -> bool: - """Check if format is user-browseable. Delegates to ytdlp helper.""" - return is_browseable_format(fmt) - - def _format_id_for_query_index( - self, - query_format: str, - url: str, - formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], - ytdlp_tool: YtDlpTool, - ) -> Optional[str]: - if not query_format or not _FORMAT_INDEX_RE.match(str(query_format)): - return None - - try: - s_val = str(query_format).strip() - idx = int(s_val.lstrip("#")) - except Exception: - raise ValueError(f"Invalid format index: {query_format}") - - fmts = self._list_formats_cached( - url, - playlist_items_value=None, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if not fmts: - raise ValueError("Unable to list formats for the URL; cannot resolve numeric format index") - - # Prioritize exact format_id match if it's a numeric string that happens to be an ID - # (e.g. YouTube's 251 for opus). - if s_val and not s_val.startswith("#"): - if any(str(f.get("format_id", "")) == s_val for f in fmts): - return s_val - - candidate_formats = [f for f in fmts if self._is_browseable_format(f)] - filtered_formats = candidate_formats if candidate_formats else list(fmts) - - if not filtered_formats: - raise ValueError("No formats available for selection") - - if idx <= 0 or idx > len(filtered_formats): - raise ValueError(f"Format index {idx} out of range (1..{len(filtered_formats)})") - - chosen = filtered_formats[idx - 1] - selection_format_id = str(chosen.get("format_id") or "").strip() - if not selection_format_id: - raise ValueError("Selected format has no format_id") - - try: - vcodec = str(chosen.get("vcodec", "none")) - acodec = str(chosen.get("acodec", "none")) - if vcodec != "none" and acodec == "none": - selection_format_id = f"{selection_format_id}+bestaudio" - except Exception: - pass - - return selection_format_id - - @staticmethod - def _canonicalize_url_for_storage(*, requested_url: str, ytdlp_tool: YtDlpTool, playlist_items: Optional[str]) -> str: - if playlist_items: - return str(requested_url) - try: - cf = None - try: - cookie_path = ytdlp_tool.resolve_cookiefile() - if cookie_path is not None and cookie_path.is_file(): - cf = str(cookie_path) - except Exception: - cf = None - - pr = probe_url(requested_url, no_playlist=False, timeout_seconds=15, cookiefile=cf) - if isinstance(pr, dict): - for key in ("webpage_url", "original_url", "url", "requested_url"): - value = pr.get(key) - if isinstance(value, str) and value.strip(): - canon = value.strip() - return canon - except Exception as e: - debug(f"[download-file] Canonicalization error for {requested_url}: {e}") - return str(requested_url) - - - def _preflight_url_duplicate( - self, - *, - storage: Any, - hydrus_available: bool, - final_output_dir: Path, - candidate_url: Optional[str] = None, - extra_urls: Optional[List[str]] = None, - **kwargs: Any, - ) -> bool: - to_check = [] - if candidate_url: - to_check.append(str(candidate_url)) - if extra_urls: - to_check.extend([str(u) for u in extra_urls if u]) - - # De-duplicate needles to avoid redundant DB searches. - seen = set() - unique_to_check = [] - for u in to_check: - if u not in seen: - unique_to_check.append(u) - seen.add(u) - - return sh.check_url_exists_in_storage( - urls=unique_to_check, - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - auto_continue_duplicates=False, - force_prompt_in_pipeline=bool(kwargs.get("force_prompt_in_pipeline")), - ) - - def _preflight_url_duplicates_bulk( - self, - *, - urls: List[str], - storage: Any, - hydrus_available: bool, - final_output_dir: Path, - **kwargs: Any, - ) -> bool: - if not urls: - return True - unique_urls = [] - seen = set() - for u in urls: - if u and u not in seen: - unique_urls.append(u) - seen.add(u) - return sh.check_url_exists_in_storage( - urls=unique_urls, - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - auto_continue_duplicates=False, - ) - - - def _maybe_show_playlist_table(self, *, url: str, ytdlp_tool: YtDlpTool) -> bool: - ctx = pipeline_context.get_stage_context() - if ctx is not None and getattr(ctx, "total_stages", 0) > 1: - return False - - try: - cf = self._cookiefile_str(ytdlp_tool) - pr = probe_url(url, no_playlist=False, timeout_seconds=15, cookiefile=cf) - except Exception: - pr = None - if not isinstance(pr, dict): - return False - entries = pr.get("entries") - if not isinstance(entries, list) or len(entries) <= 1: - return False - - extractor_name = "" - try: - extractor_name = str(pr.get("extractor") or pr.get("extractor_key") or "").strip().lower() - except Exception: - extractor_name = "" - table_type: Optional[str] = None - if "bandcamp" in extractor_name: - table_type = "bandcamp" - elif "youtube" in extractor_name: - table_type = "youtube" - - max_rows = 200 - display_entries = entries[:max_rows] - - def _entry_to_url(entry: Any) -> Optional[str]: - if not isinstance(entry, dict): - return None - for key in ("webpage_url", "original_url", "url"): - v = entry.get(key) - if isinstance(v, str) and v.strip(): - s_val = v.strip() - try: - if urlparse(s_val).scheme in {"http", "https"}: - return s_val - except Exception: - return s_val - - entry_id = entry.get("id") - if isinstance(entry_id, str) and entry_id.strip(): - extractor_name_inner = str(pr.get("extractor") or pr.get("extractor_key") or "").lower() - if "youtube" in extractor_name_inner: - return f"https://www.youtube.com/watch?v={entry_id.strip()}" - return None - - table = Table(preserve_order=True) - safe_url = str(url or "").strip() - table.title = f'download-file -url "{safe_url}"' if safe_url else "download-file" - if table_type: - try: - table.set_table(table_type) - except Exception: - table.table = table_type - table.set_source_command("download-file", []) - try: - table._perseverance(True) - except Exception: - pass - - results_list: List[Dict[str, Any]] = [] - for idx, entry in enumerate(display_entries, 1): - title = None - uploader = None - duration = None - entry_url = _entry_to_url(entry) - try: - if isinstance(entry, dict): - title = entry.get("title") - uploader = entry.get("uploader") or pr.get("uploader") - duration = entry.get("duration") - except Exception: - pass - - row = build_table_result_payload( - table="download-file", - title=str(title or f"Item {idx}"), - detail=str(uploader or ""), - columns=[ - ("#", str(idx)), - ("Title", str(title or "")), - ("Duration", str(duration or "")), - ("Uploader", str(uploader or "")), - ], - selection_args=( - ["-url", str(entry_url)] if entry_url else ["-url", str(url), "-item", str(idx)] - ), - media_kind="playlist-item", - playlist_index=idx, - url=entry_url, - target=entry_url, - ) - results_list.append(row) - table.add_result(row) - - pipeline_context.set_current_stage_table(table) - pipeline_context.set_last_result_table(table, results_list) - - try: - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm: AbstractContextManager[Any] = nullcontext() - if callable(suspend): - maybe_cm = suspend() - if maybe_cm is not None: - cm = maybe_cm # type: ignore[assignment] - with cm: - get_stderr_console().print(table) - except Exception: - pass - setattr(table, "_rendered_by_cmdlet", True) - return True - - def _maybe_show_format_table_for_single_url( - self, - *, - mode: str, - clip_spec: Any, - clip_values: Sequence[str], - playlist_items: Optional[str], - ytdl_format: Any, - supported_url: Sequence[str], - playlist_selection_handled: bool, - ytdlp_tool: YtDlpTool, - formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], - storage: Any, - hydrus_available: bool, - final_output_dir: Path, - args: Sequence[str], - skip_preflight: bool = False, - ) -> Optional[int]: - try: - ctx = pipeline_context.get_stage_context() - if ctx is not None and getattr(ctx, "total_stages", 0) > 1: - # In pipelines, skip interactive format tables; require explicit -query format. - return None - except Exception: - pass - if ( - mode != "audio" - and not clip_spec - and not clip_values - and not playlist_items - and not ytdl_format - and len(supported_url) == 1 - and not playlist_selection_handled - ): - url = supported_url[0] - - canonical_url = self._canonicalize_url_for_storage( - requested_url=url, - ytdlp_tool=ytdlp_tool, - playlist_items=playlist_items, - ) - if not skip_preflight: - if not self._preflight_url_duplicate( - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - candidate_url=canonical_url, - extra_urls=[url], - ): - log(f"Skipping download: {url}", file=sys.stderr) - return 0 - - formats = self._list_formats_cached( - url, - playlist_items_value=None, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - - if formats and len(formats) > 1: - candidate_formats = [f for f in formats if self._is_browseable_format(f)] - filtered_formats = candidate_formats if candidate_formats else list(formats) - - debug(f"Formatlist: showing {len(filtered_formats)} formats (raw={len(formats)})") - - base_cmd = f'download-file "{url}"' - remaining_args = [arg for arg in args if arg not in [url] and not arg.startswith("-")] - if remaining_args: - base_cmd += " " + " ".join(remaining_args) - - table = Table(title=f"Available formats for {url}", max_columns=10, preserve_order=True) - table.set_table("ytdlp.formatlist") - table.set_source_command("download-file", [url]) - - debug(f"[ytdlp.formatlist] Displaying format selection table for {url}") - debug("[ytdlp.formatlist] Provider: ytdlp (routing to download-file via TABLE_AUTO_STAGES)") - - results_list: List[Dict[str, Any]] = [] - for idx, fmt in enumerate(filtered_formats, 1): - resolution = fmt.get("resolution", "") - ext = fmt.get("ext", "") - vcodec = fmt.get("vcodec", "none") - acodec = fmt.get("acodec", "none") - filesize = fmt.get("filesize") - filesize_approx = fmt.get("filesize_approx") - format_id = fmt.get("format_id", "") - - selection_format_id = format_id - try: - if vcodec != "none" and acodec == "none" and format_id: - selection_format_id = f"{format_id}+bestaudio" - except Exception: - selection_format_id = format_id - - # Use ytdlp helper to format for table - format_dict = format_for_table_selection( - fmt, - url, - idx, - selection_format_id=selection_format_id, - ) - - # Add base command for display - format_dict["cmd"] = base_cmd - - def _merge_query_args(selection_args: List[str], query_value: str) -> List[str]: - if not query_value: - return selection_args - merged = list(selection_args or []) - if "-query" in merged: - idx_query = merged.index("-query") - if idx_query + 1 < len(merged): - existing = str(merged[idx_query + 1] or "").strip() - merged[idx_query + 1] = f"{existing},{query_value}" if existing else query_value - else: - merged.append(query_value) - else: - merged.extend(["-query", query_value]) - return merged - - # Append clip values to selection args if needed - selection_args: List[str] = list(format_dict.get("_selection_args") or []) - try: - if (not clip_spec) and clip_values: - clip_query = f"clip:{','.join([v for v in clip_values if v])}" - selection_args = _merge_query_args(selection_args, clip_query) - except Exception: - pass - format_dict["_selection_args"] = selection_args - - # Also update in full_metadata for provider registration - format_dict["full_metadata"]["_selection_args"] = selection_args - - results_list.append(format_dict) - table.add_result(format_dict) - - try: - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm: AbstractContextManager[Any] = nullcontext() - if callable(suspend): - maybe_cm = suspend() - if maybe_cm is not None: - cm = maybe_cm # type: ignore[assignment] - with cm: - get_stderr_console().print(table) - except Exception: - pass - - setattr(table, "_rendered_by_cmdlet", True) - pipeline_context.set_current_stage_table(table) - pipeline_context.set_last_result_table(table, results_list) - - debug(f"[ytdlp.formatlist] Format table registered with {len(results_list)} formats") - debug( - f"[ytdlp.formatlist] When user selects @N, will invoke: download-file {url} -query 'format:'" - ) - - log("", file=sys.stderr) - return 0 - - return None - - def _download_supported_urls( - self, - *, - supported_url: Sequence[str], - ytdlp_tool: YtDlpTool, - args: Sequence[str], - config: Dict[str, Any], - final_output_dir: Path, - mode: str, - clip_spec: Any, - clip_ranges: Optional[List[tuple[int, int]]], - query_hash_override: Optional[str], - embed_chapters: bool, - write_sub: bool, - quiet_mode: bool, - playlist_items: Optional[str], - ytdl_format: Any, - skip_per_url_preflight: bool, - forced_single_format_id: Optional[str], - forced_single_format_for_batch: bool, - formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], - storage: Any, - hydrus_available: bool, - download_timeout_seconds: int, - ) -> int: - downloaded_count = 0 - duplicate_skipped_count = 0 - downloaded_pipe_objects: List[Dict[str, Any]] = [] - pipe_seq = 0 - clip_sections_spec = self._build_clip_sections_spec(clip_ranges) - suppress_nested, batch_total, batch_index, batch_label = self._batch_progress_state(config) - total_urls = len(supported_url or []) - aggregate_status_mode = bool(suppress_nested or total_urls > 1) - - if clip_sections_spec: - try: - debug(f"Clip sections spec: {clip_sections_spec}") - except Exception: - pass - - for url_index, url in enumerate(supported_url, 1): - try: - display_total = batch_total if batch_total > 0 else total_urls - display_index = batch_index if batch_total > 0 else url_index - display_label = batch_label or str(url) - - canonical_url = url - if not skip_per_url_preflight or clip_ranges: - canonical_url = self._canonicalize_url_for_storage( - requested_url=url, - ytdlp_tool=ytdlp_tool, - playlist_items=playlist_items, - ) - - if not skip_per_url_preflight: - if not self._preflight_url_duplicate( - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - candidate_url=canonical_url, - extra_urls=[url], - force_prompt_in_pipeline=bool(clip_ranges), - ): - duplicate_skipped_count += 1 - log(f"Skipping download (duplicate found): {url}", file=sys.stderr) - continue - - try: - debug_panel( - f"Download item {display_index}/{display_total or total_urls}", - [ - ("url", url), - ("canonical_url", canonical_url), - ("mode", mode), - ("format", ytdl_format or "auto"), - ("duplicate_preflight", not skip_per_url_preflight), - ], - border_style="green", - ) - except Exception: - pass - - if aggregate_status_mode: - try: - if display_total > 0: - PipelineProgress(pipeline_context).set_status( - f"downloading {display_index}/{display_total}: {display_label}" - ) - except Exception: - pass - else: - PipelineProgress(pipeline_context).begin_steps(2) - - actual_format = ytdl_format - actual_playlist_items = playlist_items - - if playlist_items and not ytdl_format: - import re - - if re.search(r"[^0-9,-]", playlist_items): - actual_format = playlist_items - actual_playlist_items = None - - if mode == "audio" and not actual_format: - actual_format = "bestaudio" - - if mode == "video" and not actual_format: - configured = (ytdlp_tool.default_format("video") or "").strip() - if configured and configured != "bestvideo+bestaudio/best": - # Check if the configured default is a short-hand resolution (e.g. "720") - # and resolve it to a proper selector. - resolved = ytdlp_tool.resolve_height_selector(configured) - if resolved: - actual_format = resolved - else: - actual_format = configured - - forced_single_applied = False - if ( - forced_single_format_for_batch - and forced_single_format_id - and not ytdl_format - and not actual_playlist_items - ): - actual_format = forced_single_format_id - forced_single_applied = True - - # Proactive fallback for single audio formats which might be unstable - if ( - actual_format - and isinstance(actual_format, str) - and actual_format == "audio" - ): - actual_format = "bestaudio" - - if clip_sections_spec and mode != "audio": - clip_format_basis = actual_format - if not clip_format_basis or str(clip_format_basis).strip().lower() in { - "bestvideo+bestaudio/best", - "bestvideo+bestaudio", - "best", - "best/b", - "best/best", - "b", - }: - preferred_clip_format = str(getattr(ytdlp_tool.defaults, "format", "") or "").strip() - if preferred_clip_format and preferred_clip_format.lower() != "audio": - clip_format_basis = preferred_clip_format - else: - clip_format_basis = ytdlp_tool.default_format("video") - clip_safe_format = ytdlp_tool.resolve_clip_safe_format(clip_format_basis) - if clip_safe_format: - actual_format = clip_safe_format - - # DEBUG: Render config panel for tracking pipeline state - if is_debug_enabled(): - try: - from rich.table import Table as RichTable - from rich import box as RichBox - from tool.ytdlp import YtDlpDefaults - - t = RichTable(title="Download Config", show_header=True, header_style="bold magenta", box=RichBox.ROUNDED) - t.add_column("Property", style="cyan") - t.add_column("Value", style="green") - t.add_row("URL", url) - t.add_row("Mode", mode) - t.add_row("Format", str(actual_format)) - t.add_row("Playlist Items", str(actual_playlist_items)) - - # Browser/Cookie info from ytdlp tool - defaults = getattr(ytdlp_tool, "defaults", None) - if isinstance(defaults, YtDlpDefaults): - t.add_row("Cookie File", str(defaults.cookiefile or "None")) - t.add_row("Browser Cookies", str(defaults.cookies_from_browser or "None")) - t.add_row("User Agent", str(defaults.user_agent or "default")) - - debug(t) - except Exception: - pass - - # If no format explicitly chosen, we might want to check available formats - # and maybe show a table if multiple are available? - if ( - actual_format - and isinstance(actual_format, str) - and mode != "audio" - and not clip_sections_spec - and "+" not in actual_format - and "/" not in actual_format - and "[" not in actual_format - and actual_format not in {"best", "bv", "ba", "b"} - and not forced_single_applied - ): - try: - formats = self._list_formats_cached( - url, - playlist_items_value=actual_playlist_items, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if formats: - fmt_match = next((f for f in formats if str(f.get("format_id", "")) == actual_format), None) - if fmt_match: - vcodec = str(fmt_match.get("vcodec", "none")) - acodec = str(fmt_match.get("acodec", "none")) - if vcodec != "none" and acodec == "none": - debug(f"Selected video-only format {actual_format}; using {actual_format}+bestaudio for audio") - actual_format = f"{actual_format}+bestaudio" - except Exception as e: - pass - - - attempted_single_format_fallback = False - attempted_audio_fallback_specific = False - attempted_audio_fallback_generic = False - while True: - try: - opts = DownloadOptions( - url=url, - mode=mode, - output_dir=final_output_dir, - ytdl_format=actual_format, - cookies_path=ytdlp_tool.resolve_cookiefile(), - clip_sections=clip_sections_spec, - playlist_items=actual_playlist_items, - quiet=quiet_mode, - no_playlist=False, - embed_chapters=embed_chapters, - write_sub=write_sub, - ) - - if not aggregate_status_mode: - PipelineProgress(pipeline_context).step("downloading") - result_obj = _download_with_timeout(opts, timeout_seconds=download_timeout_seconds, config=config) - break - except DownloadError as e: - cause = getattr(e, "__cause__", None) - detail = "" - try: - detail = str(cause or "") - except Exception: - detail = "" - - detail_lc = (detail or "").lower() - msg_lc = "" - try: - msg_lc = str(e or "").lower() - except Exception: - msg_lc = "" - requested_format_unavailable = ( - "requested format is not available" in detail_lc - or "requested format is not available" in msg_lc - ) - - if requested_format_unavailable and mode == "audio": - # Level 1: Try to find a specific audio-only format from the list that hopefully works - if not attempted_audio_fallback_specific: - attempted_audio_fallback_specific = True - audio_format_id = None - try: - formats = self._list_formats_cached( - url, - playlist_items_value=actual_playlist_items, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if formats: - audio_candidates = [] - for fmt in formats: - if not isinstance(fmt, dict): - continue - vcodec = str(fmt.get("vcodec", "none")) - acodec = str(fmt.get("acodec", "none")) - if acodec != "none" and vcodec == "none": - audio_candidates.append(fmt) - - if audio_candidates: - def _score_audio(fmt: Dict[str, Any]) -> float: - score = 0.0 - # Penalize DRC formats heavily - fid = str(fmt.get("format_id") or "").lower() - if "drc" in fid: - score -= 1000.0 - - for key in ("abr", "tbr", "filesize", "filesize_approx"): - try: - val = fmt.get(key) - if isinstance(val, (int, float)): - score += float(val) - break # Use the first valid metric found - if isinstance(val, str) and val.strip().isdigit(): - score += float(val) - break - except Exception: - continue - return score - - audio_candidates.sort(key=_score_audio, reverse=True) - audio_format_id = str(audio_candidates[0].get("format_id") or "").strip() or None - except Exception: - audio_format_id = None - - if audio_format_id: - actual_format = audio_format_id - debug(f"Requested audio format not available; retrying with best specific audio format: {actual_format}") - continue - - # Level 2: Fallback to generic 'bestaudio/best' if specific selection failed or wasn't found - if not attempted_audio_fallback_generic: - attempted_audio_fallback_generic = True - if actual_format != "bestaudio/best": - actual_format = "bestaudio/best" - debug("Requested audio format not available; retrying with generic fallback: bestaudio/best") - continue - - if requested_format_unavailable and mode != "audio": - if ( - forced_single_format_for_batch - and forced_single_format_id - and not ytdl_format - and not actual_playlist_items - and not attempted_single_format_fallback - ): - attempted_single_format_fallback = True - actual_format = forced_single_format_id - debug(f"Only one format available (playlist preflight); retrying with: {actual_format}") - continue - - formats = self._list_formats_cached( - url, - playlist_items_value=actual_playlist_items, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if ( - (not attempted_single_format_fallback) - and isinstance(formats, list) - and len(formats) == 1 - and isinstance(formats[0], dict) - ): - only = formats[0] - fallback_format = str(only.get("format_id") or "").strip() - selection_format_id = fallback_format - try: - vcodec = str(only.get("vcodec", "none")) - acodec = str(only.get("acodec", "none")) - if ( - not clip_sections_spec - and vcodec != "none" - and acodec == "none" - and fallback_format - ): - selection_format_id = f"{fallback_format}+bestaudio" - except Exception: - selection_format_id = fallback_format - - if selection_format_id: - attempted_single_format_fallback = True - actual_format = selection_format_id - debug(f"Only one format available; retrying with: {actual_format}") - continue - - if formats: - formats_to_show = formats - - table = Table(title=f"Available formats for {url}", max_columns=10, preserve_order=True) - table.set_table("ytdlp.formatlist") - table.set_source_command("download-file", [url]) - - results_list: List[Dict[str, Any]] = [] - for idx, fmt in enumerate(formats_to_show, 1): - resolution = fmt.get("resolution", "") - ext = fmt.get("ext", "") - vcodec = fmt.get("vcodec", "none") - acodec = fmt.get("acodec", "none") - filesize = fmt.get("filesize") - filesize_approx = fmt.get("filesize_approx") - format_id = fmt.get("format_id", "") - - selection_format_id = format_id - try: - if vcodec != "none" and acodec == "none" and format_id: - selection_format_id = f"{format_id}+bestaudio" - except Exception: - selection_format_id = format_id - - size_str = "" - size_prefix = "" - size_bytes = filesize - if not size_bytes: - size_bytes = filesize_approx - if size_bytes: - size_prefix = "~" - try: - if isinstance(size_bytes, (int, float)) and size_bytes > 0: - size_mb = float(size_bytes) / (1024 * 1024) - size_str = f"{size_prefix}{size_mb:.1f}MB" - except Exception: - size_str = "" - - desc_parts: List[str] = [] - if resolution and resolution != "audio only": - desc_parts.append(str(resolution)) - if ext: - desc_parts.append(str(ext).upper()) - if vcodec != "none": - desc_parts.append(f"v:{vcodec}") - if acodec != "none": - desc_parts.append(f"a:{acodec}") - if size_str: - desc_parts.append(size_str) - format_desc = " | ".join(desc_parts) - - format_dict = build_table_result_payload( - table="download-file", - title=f"Format {format_id}", - detail=format_desc, - columns=[ - ("ID", format_id), - ("Resolution", resolution or "N/A"), - ("Ext", ext), - ("Size", size_str or ""), - ("Video", vcodec), - ("Audio", acodec), - ], - selection_args=["-query", f"format:{selection_format_id}"], - url=url, - target=url, - media_kind="format", - full_metadata={ - "format_id": format_id, - "url": url, - "item_selector": selection_format_id, - }, - ) - - results_list.append(format_dict) - table.add_result(format_dict) - - pipeline_context.set_current_stage_table(table) - pipeline_context.set_last_result_table(table, results_list) - - try: - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm: AbstractContextManager[Any] = nullcontext() - if callable(suspend): - maybe_cm = suspend() - if maybe_cm is not None: - cm = maybe_cm # type: ignore[assignment] - with cm: - get_stderr_console().print(table) - except Exception: - pass - - PipelineProgress(pipeline_context).step("awaiting selection") - - log("Requested format is not available; select a working format with @N", file=sys.stderr) - return 1 - - raise - - results_to_emit: List[Any] = [] - if isinstance(result_obj, list): - results_to_emit = list(result_obj) - else: - paths = getattr(result_obj, "paths", None) - if isinstance(paths, list) and paths: - for p in paths: - try: - p_path = Path(p) - except Exception: - continue - try: - if p_path.suffix.lower() in _SUBTITLE_EXTS: - continue - except Exception: - pass - if not p_path.exists() or p_path.is_dir(): - continue - try: - hv = sha256_file(p_path) - except Exception: - hv = None - results_to_emit.append( - DownloadMediaResult( - path=p_path, - info=getattr(result_obj, "info", {}) or {}, - tag=list(getattr(result_obj, "tag", []) or []), - source_url=getattr(result_obj, "source_url", None) or opts.url, - hash_value=hv, - ) - ) - else: - results_to_emit = [result_obj] - - pipe_objects: List[Dict[str, Any]] = [] - for downloaded in results_to_emit: - po = self._build_pipe_object(downloaded, url, opts) - pipe_seq += 1 - try: - po.setdefault("pipe_index", pipe_seq) - except Exception: - pass - - try: - info = downloaded.info if isinstance(getattr(downloaded, "info", None), dict) else {} - except Exception: - info = {} - chapters_text = _format_chapters_note(info) if embed_chapters else None - if chapters_text: - notes = po.get("notes") - if not isinstance(notes, dict): - notes = {} - notes.setdefault("chapters", chapters_text) - po["notes"] = notes - - if write_sub: - try: - media_path = Path(str(po.get("path") or "")) - except Exception: - media_path = None - - if media_path is not None and media_path.exists() and media_path.is_file(): - sub_path = _best_subtitle_sidecar(media_path) - if sub_path is not None: - sub_text = _read_text_file(sub_path) - if sub_text: - notes = po.get("notes") - if not isinstance(notes, dict): - notes = {} - notes["sub"] = sub_text - po["notes"] = notes - try: - sub_path.unlink() - except Exception: - pass - - pipe_objects.append(po) - - try: - if clip_ranges and len(pipe_objects) == len(clip_ranges): - source_hash = query_hash_override or self._find_existing_hash_for_url( - storage, - canonical_url, - hydrus_available=hydrus_available, - ) - self._apply_clip_decorations(pipe_objects, clip_ranges, source_king_hash=source_hash) - except Exception: - pass - - if not aggregate_status_mode: - PipelineProgress(pipeline_context).step("finalized") - - stage_ctx = pipeline_context.get_stage_context() - emit_enabled = bool(stage_ctx is not None) - for pipe_obj_dict in pipe_objects: - if emit_enabled: - pipeline_context.emit(pipe_obj_dict) - - if pipe_obj_dict.get("url"): - pipe_obj = coerce_to_pipe_object(pipe_obj_dict) - register_url_with_local_library(pipe_obj, config) - - try: - downloaded_pipe_objects.append(pipe_obj_dict) - except Exception: - pass - - downloaded_count += len(pipe_objects) - try: - debug_panel( - "download-file result", - [ - ("emitted", len(pipe_objects)), - ("url", url), - ], - border_style="green", - ) - except Exception: - pass - - except DownloadError as e: - log(f"Download failed for {url}: {e}", file=sys.stderr) - except Exception as e: - log(f"Error processing {url}: {e}", file=sys.stderr) - - if downloaded_count > 0: - return 0 - if duplicate_skipped_count > 0: - return 0 - - log("No downloads completed", file=sys.stderr) - return 1 - - def _run_streaming_urls( - self, - *, - streaming_urls: List[str], - args: Sequence[str], - config: Dict[str, Any], - parsed: Dict[str, Any], - ) -> int: - try: - suppress_nested, _batch_total, _batch_index, _batch_label = self._batch_progress_state(config) - - ytdlp_tool = YtDlpTool(config) - - raw_url = list(streaming_urls) - supported_url, unsupported_list = self._filter_supported_urls(raw_url) - - if not supported_url: - log("No yt-dlp-supported url to download", file=sys.stderr) - return 1 - - if unsupported_list: - debug(f"Skipping {len(unsupported_list)} unsupported url (use direct HTTP mode)") - - final_output_dir = resolve_target_dir(parsed, config) - if not final_output_dir: - return 1 - - progress = PipelineProgress(pipeline_context) - using_shared_ui = pipeline_context.get_stage_context() is not None - try: - # If we are already in a pipeline stage, the parent UI is already handling progress. - # Calling ensure_local_ui can cause re-initialization hangs on some platforms. - if not using_shared_ui: - progress.ensure_local_ui( - label="download-file", - total_items=len(supported_url), - items_preview=supported_url, - ) - else: - try: - if not suppress_nested: - progress.begin_pipe( - total_items=len(supported_url), - items_preview=supported_url, - ) - except Exception as err: - debug(f"[download-file] PipelineProgress begin_pipe error: {err}") - except Exception as e: - debug(f"[download-file] PipelineProgress update error: {e}") - clip_spec = parsed.get("clip") - query_spec = parsed.get("query") - - query_keyed = self._parse_query_keyed_spec(str(query_spec) if query_spec is not None else None) - - query_hash_override = self._extract_hash_override(str(query_spec) if query_spec is not None else None, query_keyed) - - embed_chapters = True - write_sub = True - - query_format: Optional[str] = None - try: - fmt_values = query_keyed.get("format", []) if isinstance(query_keyed, dict) else [] - fmt_candidate = fmt_values[-1] if fmt_values else None - if fmt_candidate is not None: - query_format = str(fmt_candidate).strip() - except Exception: - query_format = None - - query_audio: Optional[bool] = None - try: - audio_values = query_keyed.get("audio", []) if isinstance(query_keyed, dict) else [] - audio_candidate = audio_values[-1] if audio_values else None - if audio_candidate is not None: - s_val = str(audio_candidate).strip().lower() - if s_val in {"1", "true", "t", "yes", "y", "on"}: - query_audio = True - elif s_val in {"0", "false", "f", "no", "n", "off"}: - query_audio = False - elif s_val: - query_audio = True - except Exception: - query_audio = None - - query_wants_audio = False - if query_format: - try: - query_wants_audio = str(query_format).strip().lower() == "audio" - except Exception: - query_wants_audio = False - - if query_audio is not None: - wants_audio = bool(query_audio) - else: - wants_audio = bool(query_wants_audio) - mode = "audio" if wants_audio else "video" - - clip_ranges, clip_invalid, clip_values = self._parse_clip_ranges_and_apply_items( - clip_spec=str(clip_spec) if clip_spec is not None else None, - query_keyed=query_keyed, - parsed=parsed, - query_spec=str(query_spec) if query_spec is not None else None, - ) - if clip_invalid: - return 1 - - if clip_ranges: - try: - debug(f"Clip ranges: {clip_ranges}") - except Exception: - pass - - quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False - - storage, hydrus_available = self._init_storage(config if isinstance(config, dict) else {}) - - formats_cache: Dict[str, Optional[List[Dict[str, Any]]]] = {} - playlist_items = str(parsed.get("item")) if parsed.get("item") else None - ytdl_format = None - height_selector = None - if query_format and not query_wants_audio: - try: - # Always try to resolve numeric inputs like 720/1080p to height selectors. - # If it is a real format ID, resolve_height_selector will return None - # and we will fall back to using the literal format ID below. - height_selector = ytdlp_tool.resolve_height_selector(query_format) - except Exception: - height_selector = None - if query_wants_audio: - # Explicit `format:audio` must always force bestaudio selection - # and avoid format-list/selector ambiguity. - ytdl_format = "bestaudio" - elif height_selector: - ytdl_format = height_selector - elif query_format: - # Use query_format as literal format ID (e.g., from table selection like '251') - ytdl_format = query_format - - playlist_selection_handled = False - - if len(supported_url) == 1 and not playlist_items: - candidate_url = supported_url[0] - - # If query_format is provided and numeric, resolve it now. - if query_format and not query_wants_audio and not ytdl_format: - try: - idx_fmt = self._format_id_for_query_index(query_format, candidate_url, formats_cache, ytdlp_tool) - if idx_fmt: - ytdl_format = idx_fmt - except ValueError as e: - # Fallback: Treat as literal format if resolution fails or it's not a valid row index. - debug(f"Format resolution for '{query_format}' failed ({e}); treating as literal.") - ytdl_format = query_format - - if not ytdl_format: - if self._maybe_show_playlist_table(url=candidate_url, ytdlp_tool=ytdlp_tool): - playlist_selection_handled = True - # ... (existing logging code) ... - return 0 - - skip_per_url_preflight = False - try: - skip_preflight_override = bool(config.get("_skip_url_preflight")) if isinstance(config, dict) else False - except Exception: - skip_preflight_override = False - - if skip_preflight_override: - skip_per_url_preflight = True - elif len(supported_url) > 1: - if not self._preflight_url_duplicates_bulk( - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - urls=list(supported_url), - ): - return 0 - skip_per_url_preflight = True - - forced_single_format_id: Optional[str] = None - forced_single_format_for_batch = False - if len(supported_url) > 1 and not playlist_items and not ytdl_format: - try: - sample_url = str(supported_url[0]) - fmts = self._list_formats_cached( - sample_url, - playlist_items_value=None, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if isinstance(fmts, list) and len(fmts) == 1 and isinstance(fmts[0], dict): - only_id = str(fmts[0].get("format_id") or "").strip() - if only_id: - forced_single_format_id = only_id - forced_single_format_for_batch = True - debug( - f"Playlist format preflight: only one format available; using {forced_single_format_id} for all items" - ) - except Exception: - forced_single_format_id = None - forced_single_format_for_batch = False - - early_ret = self._maybe_show_format_table_for_single_url( - mode=mode, - clip_spec=clip_spec, - clip_values=clip_values, - playlist_items=playlist_items, - ytdl_format=ytdl_format, - supported_url=supported_url, - playlist_selection_handled=playlist_selection_handled, - ytdlp_tool=ytdlp_tool, - formats_cache=formats_cache, - storage=storage, - hydrus_available=hydrus_available, - final_output_dir=final_output_dir, - args=args, - skip_preflight=skip_preflight_override, - ) - if early_ret is not None: - return int(early_ret) - - # Auto-detect audio-only sources (e.g., Bandcamp albums) when no explicit format is requested. - if mode == "video" and not ytdl_format and not query_format and not query_wants_audio: - try: - sample_url = str(supported_url[0]) if supported_url else "" - fmts = self._list_formats_cached( - sample_url, - playlist_items_value=playlist_items, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if fmts: - has_video = any(str(f.get("vcodec", "none")) != "none" for f in fmts if isinstance(f, dict)) - has_audio = any(str(f.get("acodec", "none")) != "none" for f in fmts if isinstance(f, dict)) - if has_audio and not has_video: - mode = "audio" - ytdl_format = ytdlp_tool.default_format("audio") - debug("[download-file] No video formats detected; switching to audio mode") - else: - if "bandcamp.com/album/" in sample_url: - mode = "audio" - ytdl_format = ytdlp_tool.default_format("audio") - debug("[download-file] Bandcamp album detected; switching to audio mode") - except Exception as e: - debug(f"[download-file] Audio-only detection error: {e}") - - # Auto-detect audio mode for explicit format IDs - if len(supported_url) == 1 and ytdl_format and mode != "audio": - try: - candidates = self._list_formats_cached( - supported_url[0], - playlist_items_value=playlist_items, - formats_cache=formats_cache, - ytdlp_tool=ytdlp_tool, - ) - if candidates: - match = next((f for f in candidates if str(f.get("format_id", "")) == str(ytdl_format)), None) - if match: - vcodec = str(match.get("vcodec", "none")) - acodec = str(match.get("acodec", "none")) - if vcodec == "none" and acodec != "none": - debug(f"[download-file] Requested format {ytdl_format} is audio-only; switching mode to audio") - mode = "audio" - except Exception as e: - debug(f"[download-file] Error validating format mode: {e}") - - timeout_seconds = 300 - try: - override = config.get("_pipeobject_timeout_seconds") if isinstance(config, dict) else None - if override is not None: - timeout_seconds = max(1, int(override)) - except Exception: - timeout_seconds = 300 - - try: - debug_panel( - "Streaming download", - [ - ("urls", len(supported_url)), - ("mode", mode), - ("format", ytdl_format or "auto"), - ("output_dir", final_output_dir), - ("ui", "shared pipeline" if using_shared_ui else "local"), - ("playlist_items", playlist_items), - ("skip_preflight", skip_per_url_preflight), - ("timeout_seconds", timeout_seconds), - ], - border_style="blue", - ) - except Exception: - pass - return self._download_supported_urls( - supported_url=supported_url, - ytdlp_tool=ytdlp_tool, - args=args, - config=config, - final_output_dir=final_output_dir, - mode=mode, - clip_spec=clip_spec, - clip_ranges=clip_ranges, - query_hash_override=query_hash_override, - embed_chapters=embed_chapters, - write_sub=write_sub, - quiet_mode=quiet_mode, - playlist_items=playlist_items, - ytdl_format=ytdl_format, - skip_per_url_preflight=skip_per_url_preflight, - forced_single_format_id=forced_single_format_id, - forced_single_format_for_batch=forced_single_format_for_batch, - formats_cache=formats_cache, - storage=storage, - hydrus_available=hydrus_available, - download_timeout_seconds=timeout_seconds, - ) - - except Exception as e: - log(f"Error in streaming download handler: {e}", file=sys.stderr) - return 1 - - def _parse_time_ranges(self, spec: str) -> List[tuple[int, int]]: - def _to_seconds(ts: str) -> Optional[int]: - ts = str(ts).strip() - if not ts: - return None - - try: - unit_match = re.fullmatch(r"(?i)\s*(?:(?P\d+)h)?\s*(?:(?P\d+)m)?\s*(?:(?P\d+(?:\.\d+)?)s)?\s*", ts) - except Exception: - unit_match = None - if unit_match and unit_match.group(0).strip() and any(unit_match.group(g) for g in ("h", "m", "s")): - try: - hours = int(unit_match.group("h") or 0) - minutes = int(unit_match.group("m") or 0) - seconds = float(unit_match.group("s") or 0) - total = (hours * 3600) + (minutes * 60) + seconds - return int(total) - except Exception: - return None - - if ":" in ts: - parts = [p.strip() for p in ts.split(":")] - if len(parts) == 2: - hh_s = "0" - mm_s, ss_s = parts - elif len(parts) == 3: - hh_s, mm_s, ss_s = parts - else: - return None - - try: - hours = int(hh_s) - minutes = int(mm_s) - seconds = float(ss_s) - total = (hours * 3600) + (minutes * 60) + seconds - return int(total) - except Exception: - return None - - try: - return int(float(ts)) - except Exception: - return None - - ranges: List[tuple[int, int]] = [] - if not spec: - return ranges - - for piece in str(spec).split(","): - piece = piece.strip() - if not piece: - continue - if "-" not in piece: - return [] - start_s, end_s = [p.strip() for p in piece.split("-", 1)] - start = _to_seconds(start_s) - end = _to_seconds(end_s) - if start is None or end is None or start >= end: - return [] - ranges.append((start, end)) - - return ranges - - @staticmethod - def _parse_keyed_csv_spec(spec: str, *, default_key: str) -> Dict[str, List[str]]: - out: Dict[str, List[str]] = {} - if not isinstance(spec, str): - spec = str(spec) - text = spec.strip() - if not text: - return out - - active = (default_key or "").strip().lower() or "clip" - key_pattern = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*(.*)$") - - for raw_piece in text.split(","): - piece = raw_piece.strip() - if not piece: - continue - - m = key_pattern.match(piece) - if m: - active = (m.group(1) or "").strip().lower() or active - value = (m.group(2) or "").strip() - if value: - out.setdefault(active, []).append(value) - continue - - out.setdefault(active, []).append(piece) - - return out - - def _build_clip_sections_spec(self, clip_ranges: Optional[List[tuple[int, int]]]) -> Optional[str]: - ranges: List[str] = [] - if clip_ranges: - for start_s, end_s in clip_ranges: - ranges.append(f"{start_s}-{end_s}") - return ",".join(ranges) if ranges else None - - def _build_pipe_object(self, download_result: Any, url: str, opts: DownloadOptions) -> Dict[str, Any]: - info: Dict[str, Any] = download_result.info if isinstance(download_result.info, dict) else {} - media_path = Path(download_result.path) - hash_value = download_result.hash_value or sha256_file(media_path) - title = info.get("title") or media_path.stem - tag = list(download_result.tag or []) - - if title and f"title:{title}" not in tag: - tag.insert(0, f"title:{title}") - - final_url = None - try: - page_url = info.get("webpage_url") or info.get("original_url") or info.get("url") - if page_url: - final_url = str(page_url) - except Exception: - final_url = None - if not final_url and url: - final_url = str(url) - - return build_file_result_payload( - title=title, - path=str(media_path), - hash_value=hash_value, - url=final_url, - tag=tag, - store=getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH", - action="cmdlet:download-file", - is_temp=True, - ytdl_format=getattr(opts, "ytdl_format", None), - media_kind="video" if opts.mode == "video" else "audio", - ) - - @staticmethod - def download_streaming_url_as_pipe_objects( - url: str, - config: Dict[str, Any], - *, - mode_hint: Optional[str] = None, - ytdl_format_hint: Optional[str] = None, - ) -> List[Dict[str, Any]]: - """Download a yt-dlp-supported URL and return PipeObject-style dict(s). - - This is a lightweight helper intended for cmdlets that need to expand streaming URLs - into local files without re-implementing yt-dlp glue. - """ - url_str = str(url or "").strip() - if not url_str: - return [] - - if not is_url_supported_by_ytdlp(url_str): - return [] - - try: - from SYS.config import resolve_output_dir - - out_dir = resolve_output_dir(config) - if out_dir is None: - return [] - except Exception: - return [] - - cookies_path = None - try: - cookie_candidate = YtDlpTool(config).resolve_cookiefile() - if cookie_candidate is not None and cookie_candidate.is_file(): - cookies_path = cookie_candidate - except Exception: - cookies_path = None - - quiet_download = False - try: - quiet_download = bool((config or {}).get("_quiet_background_output")) - except Exception: - quiet_download = False - - mode = str(mode_hint or "").strip().lower() if mode_hint else "" - if mode not in {"audio", "video"}: - mode = "video" - try: - cf = ( - str(cookies_path) - if cookies_path is not None and cookies_path.is_file() else None - ) - fmts_probe = list_formats( - url_str, - no_playlist=False, - playlist_items=None, - cookiefile=cf, - ) - if isinstance(fmts_probe, list) and fmts_probe: - has_video = False - for f in fmts_probe: - if not isinstance(f, dict): - continue - vcodec = str(f.get("vcodec", "none") or "none").strip().lower() - if vcodec and vcodec != "none": - has_video = True - break - mode = "video" if has_video else "audio" - except Exception: - mode = "video" - - fmt_hint = str(ytdl_format_hint).strip() if ytdl_format_hint else "" - chosen_format: Optional[str] - if fmt_hint: - chosen_format = fmt_hint - else: - chosen_format = None - if mode == "audio": - chosen_format = "bestaudio" - - opts = DownloadOptions( - url=url_str, - mode=mode, - output_dir=Path(out_dir), - cookies_path=cookies_path, - ytdl_format=chosen_format, - quiet=quiet_download, - embed_chapters=True, - write_sub=True, - ) - - try: - result_obj = _download_with_timeout(opts, timeout_seconds=300, config=config) - except Exception as exc: - log(f"[download-file] Download failed for {url_str}: {exc}", file=sys.stderr) - return [] - - results: List[Any] - if isinstance(result_obj, list): - results = list(result_obj) - else: - paths = getattr(result_obj, "paths", None) - if isinstance(paths, list) and paths: - results = [] - for p in paths: - try: - p_path = Path(p) - except Exception: - continue - if not p_path.exists() or p_path.is_dir(): - continue - try: - hv = sha256_file(p_path) - except Exception: - hv = None - try: - results.append( - DownloadMediaResult( - path=p_path, - info=getattr(result_obj, "info", {}) or {}, - tag=list(getattr(result_obj, "tag", []) or []), - source_url=getattr(result_obj, "source_url", None) or url_str, - hash_value=hv, - ) - ) - except Exception: - continue - else: - results = [result_obj] - - out: List[Dict[str, Any]] = [] - for downloaded in results: - try: - info = ( - downloaded.info - if isinstance(getattr(downloaded, "info", None), dict) else {} - ) - except Exception: - info = {} - - try: - media_path = Path(str(getattr(downloaded, "path", "") or "")) - except Exception: - continue - if not media_path.exists() or media_path.is_dir(): - continue - - try: - hash_value = getattr(downloaded, "hash_value", None) or sha256_file(media_path) - except Exception: - hash_value = None - - title = None - try: - title = info.get("title") - except Exception: - title = None - title = title or media_path.stem - - tags = list(getattr(downloaded, "tag", []) or []) - if title and f"title:{title}" not in tags: - tags.insert(0, f"title:{title}") - - final_url = None - try: - page_url = info.get("webpage_url") or info.get("original_url") or info.get("url") - if page_url: - final_url = str(page_url) - except Exception: - final_url = None - if not final_url: - final_url = url_str - - po: Dict[str, Any] = { - "path": str(media_path), - "hash": hash_value, - "title": title, - "url": final_url, - "tag": tags, - "action": "cmdlet:download-file", - "is_temp": True, - "ytdl_format": getattr(opts, "ytdl_format", None), - "store": getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH", - "media_kind": "video" if opts.mode == "video" else "audio", - } - - try: - chapters_text = _format_chapters_note(info) - except Exception: - chapters_text = None - if chapters_text: - notes = po.get("notes") - if not isinstance(notes, dict): - notes = {} - notes.setdefault("chapters", chapters_text) - po["notes"] = notes - - try: - sub_path = _best_subtitle_sidecar(media_path) - except Exception: - sub_path = None - if sub_path is not None: - sub_text = _read_text_file(sub_path) - if sub_text: - notes = po.get("notes") - if not isinstance(notes, dict): - notes = {} - notes["sub"] = sub_text - po["notes"] = notes - try: - sub_path.unlink() - except Exception: - pass - - out.append(po) - - return out - @classmethod def _extract_hash_from_search_hit(cls, hit: Any) -> Optional[str]: if not isinstance(hit, dict): @@ -3001,6 +1210,8 @@ class Download_File(Cmdlet): # Parse arguments parsed = parse_cmdlet_args(args, self) + registry = self._load_provider_registry() + selection_url_prefixes = self._selection_url_prefixes(registry) # Resolve URLs from -url or positional arguments url_candidates = parsed.get("url") or [ @@ -3034,12 +1245,12 @@ class Download_File(Cmdlet): try: normalized_args, _normalized_action, item_url = extract_selection_fields( item, - extra_url_prefixes=_ALLDEBRID_PREFIXES, + extra_url_prefixes=selection_url_prefixes, ) if normalized_args: if selection_args_have_url( normalized_args, - extra_url_prefixes=_ALLDEBRID_PREFIXES, + extra_url_prefixes=selection_url_prefixes, ): selection_runs.append(list(normalized_args)) handled = True @@ -3047,7 +1258,11 @@ class Download_File(Cmdlet): selection_runs.append([str(item_url)] + list(normalized_args)) handled = True except Exception as e: - debug(f"[ytdlp] Error handling selection args: {e}") + debug_panel( + "download-file selection args failed", + [("error", e)], + border_style="yellow", + ) handled = False if not handled: residual_items.append(item) @@ -3057,7 +1272,7 @@ class Download_File(Cmdlet): for run_args in selection_runs: for u in extract_urls_from_selection_args( run_args, - extra_url_prefixes=_ALLDEBRID_PREFIXES, + extra_url_prefixes=selection_url_prefixes, ): if u not in selection_urls: selection_urls.append(u) @@ -3104,7 +1319,10 @@ class Download_File(Cmdlet): last_code = 0 total_selection = len(selection_runs) preview_items = list(selection_urls[:5]) or [ - self._selection_run_label(run_args) + self._selection_run_label( + run_args, + extra_url_prefixes=selection_url_prefixes, + ) for run_args in selection_runs[:5] ] try: @@ -3122,11 +1340,11 @@ class Download_File(Cmdlet): ) except Exception: pass - debug(f"[download-file] Processing {total_selection} selected item(s) from table...") for idx, run_args in enumerate(selection_runs, 1): - run_label = self._selection_run_label(run_args) - debug(f"[download-file] Item {idx}/{total_selection}: {run_args}") - debug("[download-file] Re-invoking download-file for selected item...") + run_label = self._selection_run_label( + run_args, + extra_url_prefixes=selection_url_prefixes, + ) try: progress.set_status( f"downloading {idx}/{total_selection}: {run_label}" @@ -3203,11 +1421,13 @@ class Download_File(Cmdlet): candidate = str(raw_url[0] or "").strip() low = candidate.lower() looks_like_url = low.startswith( - ("http://", "https://", "ftp://", "magnet:", "torrent:") + _ALLDEBRID_PREFIXES + ("http://", "https://", "ftp://", "magnet:", "torrent:") + + tuple(selection_url_prefixes) ) looks_like_provider = ( ":" in candidate and not candidate.startswith( - ("http:", "https:", "ftp:", "ftps:", "file:") + _ALLDEBRID_PREFIXES + ("http:", "https:", "ftp:", "ftps:", "file:") + + tuple(selection_url_prefixes) ) ) looks_like_windows_path = ( @@ -3226,8 +1446,6 @@ class Download_File(Cmdlet): log("No url or piped items to download", file=sys.stderr) return 1 - registry = self._load_provider_registry() - # Provider-pre-check (e.g. Internet Archive format picker) picker_result = self._maybe_show_provider_picker( raw_urls=raw_url, @@ -3239,44 +1457,6 @@ class Download_File(Cmdlet): if picker_result is not None: return int(picker_result) - provider_url_matches = self._match_provider_urls(raw_url, registry) - streaming_candidates = [ - url for url in raw_url - if provider_url_matches.get(str(url).strip()) == "ytdlp" - ] - supported_streaming, unsupported_streaming = self._filter_supported_urls(streaming_candidates) - matched_ytdlp = bool(streaming_candidates) - - streaming_exit_code: Optional[int] = None - streaming_downloaded = 0 - if supported_streaming: - streaming_exit_code = self._run_streaming_urls( - streaming_urls=supported_streaming, - args=args, - config=config, - parsed=parsed, - ) - if streaming_exit_code == 0: - streaming_downloaded += 1 - # Only remove URLs from further processing when streaming succeeded. - raw_url = [u for u in raw_url if u not in supported_streaming] - if not raw_url and not unsupported_streaming: - piped_items = [] - - if not raw_url and not piped_items: - return int(streaming_exit_code or 0) - else: - try: - skip_direct = bool(config.get("_skip_direct_on_streaming_failure")) if isinstance(config, dict) else False - except Exception: - skip_direct = False - if matched_ytdlp: - skip_direct = True - if skip_direct: - raw_url = [u for u in raw_url if u not in supported_streaming] - if not raw_url and not piped_items: - return int(streaming_exit_code or 1) - # Re-check picker if partial processing occurred picker_result = self._maybe_show_provider_picker( raw_urls=raw_url, @@ -3298,14 +1478,13 @@ class Download_File(Cmdlet): "download-file plan", [ ("output_dir", final_output_dir), - ("streaming_urls", len(supported_streaming)), ("remaining_urls", len(raw_url)), ("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))), ], border_style="cyan", ) except Exception: - debug(f"Output directory: {final_output_dir}") + pass # If the caller isn't running the shared pipeline Live progress UI (e.g. direct # cmdlet execution), start a minimal local pipeline progress panel so downloads @@ -3328,6 +1507,8 @@ class Download_File(Cmdlet): quiet_mode=quiet_mode, registry=registry, progress=progress, + parsed=parsed, + args=args, context_items=(result if isinstance(result, list) else ([result] if result else [])), ) downloaded_count += int(urls_downloaded) @@ -3344,18 +1525,11 @@ class Download_File(Cmdlet): ) downloaded_count += provider_downloaded - if downloaded_count > 0 or streaming_downloaded > 0 or magnet_submissions > 0: + if downloaded_count > 0 or magnet_submissions > 0: # Render detail panels for downloaded items when download-file is the last stage. self._maybe_render_download_details(config=config) - msg = f"✓ Successfully processed {downloaded_count} file(s)" - if magnet_submissions: - msg += f" and queued {magnet_submissions} magnet(s)" - debug(msg) return 0 - if streaming_exit_code is not None: - return int(streaming_exit_code) - log("No downloads completed", file=sys.stderr) return 1 @@ -3397,8 +1571,8 @@ class Download_File(Cmdlet): if not target_url: return None - match_provider_name_for_url = registry.get("match_provider_name_for_url") - get_provider = registry.get("get_provider") + match_provider_name_for_url = registry.get("match_plugin_name_for_url") + get_provider = registry.get("get_plugin") provider_name = None if match_provider_name_for_url: @@ -3422,7 +1596,15 @@ class Download_File(Cmdlet): if res is not None: return int(res) except Exception as e: - debug(f"Provider {provider_name} picker error: {e}") + debug_panel( + "download-file picker error", + [ + ("plugin", provider_name), + ("url", target_url), + ("error", e), + ], + border_style="yellow", + ) return None diff --git a/cmdlet/get_tag.py b/cmdlet/get_tag.py index 7e0d975..86a2044 100644 --- a/cmdlet/get_tag.py +++ b/cmdlet/get_tag.py @@ -15,12 +15,13 @@ import sys from SYS.logger import log, debug from Provider.metadata_provider import ( + get_default_subject_scrape_provider, get_metadata_provider, + get_metadata_provider_for_url, list_metadata_providers, scrape_isbn_metadata, scrape_openlibrary_metadata, ) -import subprocess from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -64,133 +65,6 @@ def _dedup_tags_preserve_order(tags: List[str]) -> List[str]: return out -def _resolve_candidate_urls_for_item( - result: Any, - backend: Any, - file_hash: str, - config: Dict[str, - Any], -) -> List[str]: - """Get candidate URLs from backend and/or piped result.""" - try: - from SYS.metadata import normalize_urls - except Exception: - normalize_urls = None # type: ignore[assignment] - - urls: List[str] = [] - # 1) Backend URL association (best source of truth) - try: - backend_urls = backend.get_url(file_hash, config=config) - if backend_urls: - if normalize_urls: - urls.extend(normalize_urls(backend_urls)) - else: - urls.extend( - [ - str(u).strip() for u in backend_urls - if isinstance(u, str) and str(u).strip() - ] - ) - except Exception: - pass - - # 2) Backend metadata url field - try: - meta = backend.get_metadata(file_hash, config=config) - if isinstance(meta, dict) and meta.get("url"): - if normalize_urls: - urls.extend(normalize_urls(meta.get("url"))) - else: - raw = meta.get("url") - if isinstance(raw, list): - urls.extend( - [ - str(u).strip() for u in raw - if isinstance(u, str) and str(u).strip() - ] - ) - elif isinstance(raw, str) and raw.strip(): - urls.append(raw.strip()) - except Exception: - pass - - # 3) Piped result fields - def _get(obj: Any, key: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - for key in ("url", "webpage_url", "source_url", "target"): - val = _get(result, key, None) - if not val: - continue - if normalize_urls: - urls.extend(normalize_urls(val)) - continue - if isinstance(val, str) and val.strip(): - urls.append(val.strip()) - elif isinstance(val, list): - urls.extend( - [str(u).strip() for u in val if isinstance(u, str) and str(u).strip()] - ) - - meta_field = _get(result, "metadata", None) - if isinstance(meta_field, dict) and meta_field.get("url"): - val = meta_field.get("url") - if normalize_urls: - urls.extend(normalize_urls(val)) - elif isinstance(val, list): - urls.extend( - [str(u).strip() for u in val if isinstance(u, str) and str(u).strip()] - ) - elif isinstance(val, str) and val.strip(): - urls.append(val.strip()) - - # Dedup - return _dedup_tags_preserve_order(urls) - - -def _pick_supported_ytdlp_url(urls: List[str]) -> Optional[str]: - """Pick the first URL that looks supported by yt-dlp (best effort).""" - if not urls: - return None - - def _is_hydrus_file_url(u: str) -> bool: - text = str(u or "").strip().lower() - if not text: - return False - # Hydrus-local file URLs are retrievable blobs, not original source pages. - # yt-dlp generally can't extract meaningful metadata from these. - return ("/get_files/file" in text) and ("hash=" in text) - - http_urls: List[str] = [] - for u in urls: - text = str(u or "").strip() - if text.lower().startswith(("http://", "https://")): - http_urls.append(text) - - # Prefer non-Hydrus URLs for yt-dlp scraping. - candidates = [u for u in http_urls if not _is_hydrus_file_url(u)] - if not candidates: - return None - - # Prefer a true support check when the Python module is available. - try: - from tool.ytdlp import is_url_supported_by_ytdlp - - for text in candidates: - try: - if is_url_supported_by_ytdlp(text): - return text - except Exception: - continue - except Exception: - pass - - # Fallback: use the first non-Hydrus http(s) URL and let extraction decide. - return candidates[0] if candidates else None - - # Tag item for ResultTable display and piping from dataclasses import dataclass @@ -211,15 +85,12 @@ class TagItem: service_name: Optional[str] = None path: Optional[str] = None - def __post_init__(self): - # Make ResultTable happy by adding standard fields - # NOTE: Don't set 'title' - we want only the tag column in ResultTable + def __post_init__(self) -> None: self.detail = f"Tag #{self.tag_index}" self.target = self.tag_name self.media_kind = "tag" def to_dict(self) -> Dict[str, Any]: - """Convert to dict for JSON serialization.""" return { "tag_name": self.tag_name, "tag_index": self.tag_index, @@ -230,281 +101,42 @@ class TagItem: } -def _emit_tags_as_table( - tags_list: List[str], - file_hash: Optional[str], - store: str = "hydrus", - service_name: Optional[str] = None, - config: Optional[Dict[str, - Any]] = None, - item_title: Optional[str] = None, - path: Optional[str] = None, - subject: Optional[Any] = None, - quiet: bool = False, -) -> None: - """Emit tags as TagItem objects and display via ResultTable. - - Displays tags in a rich detail panel with file context (hash, title, URL, etc). - Creates a table of individual tag items to allow selection and downstream piping. - Preserves all metadata from subject (URLs, extensions, etc.) through to display. - - Makes tags @-selectable via ctx.set_last_result_table() for chaining: - - get-tag @1 | delete-tag (remove a specific tag) - - get-tag @2 | add-url (add URL to tagged file) - - Args: - tags_list: List of tag strings to display - file_hash: SHA256 hash of file - store: Backend name (e.g., "hydrus", "local", "url") - service_name: Tag service name (if from Hydrus) - config: Application configuration - item_title: Optional file title to display - path: Optional file path - subject: Full context object (should preserve original metadata) - quiet: If True, don't display (emit-only mode) - """ - metadata = prepare_detail_metadata( - subject, - include_subject_fields=True, - title=item_title, - hash_value=file_hash, - store=(service_name if service_name else store), - path=path, - ) - - # Create ItemDetailView with exclude_tags=True so the panel shows file info - # but doesn't duplicate the tag list that we show as a table below. - table = create_detail_view( - "Tags", - metadata, - max_columns=1, - exclude_tags=True, - source_command=("get-tag", []), - ) - - # Create TagItem for each tag and add to table - tag_items = [] - for idx, tag_name in enumerate(tags_list, start=1): - tag_item = TagItem( - tag_name=tag_name, - tag_index=idx, - hash=file_hash, - store=store, - service_name=service_name, - path=path, - ) - tag_items.append(tag_item) - table.add_result(tag_item) - # Also emit to pipeline for downstream processing - ctx.emit(tag_item) - - # Mark that items were already added to the table - setattr(table, "_items_added", True) - - # Display the table and persist for @N selection - if not quiet: - try: - from SYS.rich_display import stdout_console - stdout_console().print(table) - except Exception: - pass - - # Use the shared helper to persist the table for @N selection - try: - from cmdlet._shared import display_and_persist_items - # Skip panel rendering since table already exists with custom ItemDetailView - display_and_persist_items( - tag_items, - title=table.title if hasattr(table, 'title') else "Tags", - subject=subject, - display_type="custom", - table=table, - ) - except Exception: - pass - - # Also update the current stage table for TUI - try: - if hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(table) - except Exception: - pass - # Note: CLI will handle displaying the table via ResultTable formatting - - -def _filter_scraped_tags(tags: List[str]) -> List[str]: - """Filter out tags we don't want to import from scraping.""" - blocked = {"title", - "artist", - "source"} - out: List[str] = [] - seen: set[str] = set() - for t in tags: - if not t: - continue - s = str(t).strip() - if not s: - continue - ns = s.split(":", 1)[0].strip().lower() if ":" in s else "" - if ns in blocked: - continue - key = s.lower() - if key in seen: - continue - seen.add(key) - out.append(s) - return out - - -def _summarize_tags(tags_list: List[str], limit: int = 8) -> str: - """Create a summary of tags for display.""" - shown = [t for t in tags_list[:limit] if t] - summary = ", ".join(shown) - remaining = max(0, len(tags_list) - len(shown)) - if remaining > 0: - summary = f"{summary} (+{remaining} more)" if summary else f"(+{remaining} more)" - if len(summary) > 200: - summary = summary[:197] + "..." - return summary - - -def _extract_title_from(tags_list: List[str]) -> Optional[str]: - """Extract title from tags list.""" - if extract_title: - try: - return extract_title(tags_list) - except Exception: - pass - return extract_title_tag_value(tags_list) - - -def _rename_file_if_title_tag(media: Optional[Path], tags_added: List[str]) -> bool: - """Rename a local file if title: tag was added. - - Returns True if file was renamed, False otherwise. - """ - if not media or not tags_added: - return False - - # Check if any of the added tags is a title: tag - title_value = None - for tag in tags_added: - if isinstance(tag, str): - lower_tag = tag.lower() - if lower_tag.startswith("title:"): - title_value = tag.split(":", 1)[1].strip() - break - - if not title_value: - return False - - try: - # Get current file path - file_path = media - if not file_path.exists(): - return False - - # Parse file path - dir_path = file_path.parent - old_name = file_path.name - - # Get file extension - suffix = file_path.suffix or "" - - # Sanitize title for use as filename - import re - - safe_title = re.sub(r'[<>:"/\\|?*]', "", title_value).strip() - if not safe_title: - return False - - new_name = safe_title + suffix - new_file_path = dir_path / new_name - - if new_file_path == file_path: - return False - - # Build sidecar paths BEFORE renaming the file - old_sidecar = Path(str(file_path) + ".tag") - new_sidecar = Path(str(new_file_path) + ".tag") - - # Rename file - try: - file_path.rename(new_file_path) - log(f"Renamed file: {old_name} → {new_name}") - - # Rename .tag sidecar if it exists - if old_sidecar.exists(): - try: - old_sidecar.rename(new_sidecar) - log(f"Renamed sidecar: {old_name}.tag → {new_name}.tag") - except Exception as e: - log(f"Failed to rename sidecar: {e}", file=sys.stderr) - - return True - except Exception as e: - log(f"Failed to rename file: {e}", file=sys.stderr) - return False - except Exception as e: - log(f"Error during file rename: {e}", file=sys.stderr) - return False - - -def _apply_result_updates_from_tags(result: Any, tag_list: List[str]) -> None: - """Update result object with title and tag summary from tags.""" - try: - new_title = _extract_title_from(tag_list) - if new_title: - setattr(result, "title", new_title) - setattr(result, "tag_summary", _summarize_tags(tag_list)) - except Exception: - pass - - def _emit_tag_payload( source: str, tags_list: List[str], *, - hash_value: Optional[str], - extra: Optional[Dict[str, - Any]] = None, + hash_value: Optional[str] = None, store_label: Optional[str] = None, + extra: Optional[Dict[str, Any]] = None, ) -> int: - """Emit tag values as structured payload to pipeline.""" - payload: Dict[str, - Any] = { - "source": source, - "tag": list(tags_list), - "count": len(tags_list), - } - if hash_value: - payload["hash"] = hash_value - if extra: - for key, value in extra.items(): - if value is not None: - payload[key] = value - label = None - if store_label: - label = store_label - elif ctx.get_stage_context() is not None: + tags = [str(tag).strip() for tag in tags_list or [] if str(tag or "").strip()] + payload: Dict[str, Any] = { + "source": str(source or "").strip() or "tag", + "tag": tags, + "tags": list(tags), + "hash": hash_value, + } + if isinstance(extra, dict) and extra: + payload["extra"] = dict(extra) + + label = str(store_label or "").strip() if store_label else "" + if not label and ctx.get_stage_context() is not None: label = "tag" if label: ctx.store_value(label, payload) - # Emit individual TagItem objects so they can be selected by bare index - # When in pipeline, emit individual TagItem objects if ctx.get_stage_context() is not None: - for idx, tag_name in enumerate(tags_list, start=1): - tag_item = TagItem( - tag_name=tag_name, - tag_index=idx, - hash=hash_value, - store=source, - service_name=None + for idx, tag_name in enumerate(tags, start=1): + ctx.emit( + TagItem( + tag_name=tag_name, + tag_index=idx, + hash=hash_value, + store=str(source or "tag"), + service_name=None, + ) ) - ctx.emit(tag_item) else: - # When not in pipeline, just emit the payload ctx.emit(payload) return 0 @@ -576,286 +208,11 @@ def _extract_tag_value(tags_list: List[str], namespace: str) -> Optional[str]: return None -def _scrape_url_metadata( - url: str, -) -> Tuple[Optional[str], - List[str], - List[Tuple[str, - str]], - List[Dict[str, - Any]]]: - """Scrape metadata from a URL using yt-dlp. - - Returns: - (title, tags, formats, playlist_items) tuple where: - - title: Video/content title - - tags: List of extracted tags (both namespaced and freeform) - - formats: List of (display_label, format_id) tuples - - playlist_items: List of playlist entry dicts (empty if not a playlist) - """ +def _scrape_openlibrary_metadata(olid: str) -> List[str]: try: - import json as json_module - - try: - from SYS.yt_metadata import extract_ytdlp_tags - except ImportError: - extract_ytdlp_tags = None - - # Build yt-dlp command with playlist support - # IMPORTANT: Do NOT use --flat-playlist! It strips metadata like artist, album, uploader, genre - # Without it, yt-dlp gives us full metadata in an 'entries' array within a single JSON object - # This ensures we get album-level metadata from sources like BandCamp, YouTube Music, etc. - cmd = [ - "yt-dlp", - "-j", # Output JSON - "--no-warnings", - "--playlist-items", - "1-10", # Get first 10 items if it's a playlist (provides entries) - "-f", - "best", - url, - ] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - log(f"yt-dlp error: {result.stderr}", file=sys.stderr) - return None, [], [], [] - - # Parse JSON output - WITHOUT --flat-playlist, we get ONE JSON object with 'entries' array - # This gives us full metadata instead of flat format - lines = result.stdout.strip().split("\n") - if not lines or not lines[0]: - log("yt-dlp returned empty output", file=sys.stderr) - return None, [], [], [] - - # Parse the single JSON object - try: - data = json_module.loads(lines[0]) - except json_module.JSONDecodeError as e: - log(f"Failed to parse yt-dlp JSON: {e}", file=sys.stderr) - return None, [], [], [] - - # Extract title - use the main title - title = data.get("title", "Unknown") - - # Determine if this is a playlist/album (has entries array) - # is_playlist = 'entries' in data and isinstance(data.get('entries'), list) - - # Extract tags and playlist items - tags = [] - playlist_items = [] - - # IMPORTANT: Extract album/playlist-level tags FIRST (before processing entries) - # This ensures we get metadata about the collection, not just individual tracks - if extract_ytdlp_tags: - album_tags = extract_ytdlp_tags(data) - tags.extend(album_tags) - - # Case 1: Entries are nested in the main object (standard playlist structure) - if "entries" in data and isinstance(data.get("entries"), list): - entries = data["entries"] - # Build playlist items with title and duration - for idx, entry in enumerate(entries, 1): - if isinstance(entry, dict): - item_title = entry.get("title", entry.get("id", f"Track {idx}")) - item_duration = entry.get("duration", 0) - playlist_items.append( - { - "index": idx, - "id": entry.get("id", - f"track_{idx}"), - "title": item_title, - "duration": item_duration, - "url": entry.get("url") or entry.get("webpage_url", - ""), - } - ) - - # Extract tags from each entry and merge (but don't duplicate album-level tags) - # Only merge entry tags that are multi-value prefixes (not single-value like title:, artist:, etc.) - if extract_ytdlp_tags: - entry_tags = extract_ytdlp_tags(entry) - - # Single-value namespaces that should not be duplicated from entries - single_value_namespaces = { - "title", - "artist", - "album", - "creator", - "channel", - "release_date", - "upload_date", - "license", - "location", - } - - for tag in entry_tags: - # Extract the namespace (part before the colon) - tag_namespace = tag.split(":", - 1)[0].lower( - ) if ":" in tag else None - - # Skip if this namespace already exists in tags (from album level) - if tag_namespace and tag_namespace in single_value_namespaces: - # Check if any tag with this namespace already exists in tags - already_has_namespace = any( - t.split(":", - 1)[0].lower() == tag_namespace for t in tags - if ":" in t - ) - if already_has_namespace: - continue # Skip this tag, keep the album-level one - - if tag not in tags: # Avoid exact duplicates - tags.append(tag) - - # Case 2: Playlist detected by playlist_count field (BandCamp albums, etc.) - # These need a separate call with --flat-playlist to get the actual entries - elif (data.get("playlist_count") or 0) > 0 and "entries" not in data: - try: - # Make a second call with --flat-playlist to get the actual tracks - flat_cmd = [ - "yt-dlp", - "-j", - "--no-warnings", - "--flat-playlist", - "-f", - "best", - url - ] - flat_result = subprocess.run( - flat_cmd, - capture_output=True, - text=True, - timeout=30 - ) - if flat_result.returncode == 0: - flat_lines = flat_result.stdout.strip().split("\n") - # With --flat-playlist, each line is a separate track JSON object - # (not nested in a playlist container), so process ALL lines - for idx, line in enumerate(flat_lines, 1): - if line.strip().startswith("{"): - try: - entry = json_module.loads(line) - item_title = entry.get( - "title", - entry.get("id", - f"Track {idx}") - ) - item_duration = entry.get("duration", 0) - playlist_items.append( - { - "index": - idx, - "id": - entry.get("id", - f"track_{idx}"), - "title": - item_title, - "duration": - item_duration, - "url": - entry.get("url") - or entry.get("webpage_url", - ""), - } - ) - except json_module.JSONDecodeError: - pass - except Exception: - pass # Silently ignore if we can't get playlist entries - - # Fallback: if still no tags detected, get from first item - if not tags and extract_ytdlp_tags: - tags = extract_ytdlp_tags(data) - - # Extract formats from the main data object - formats = [] - if "formats" in data: - formats = _extract_url_formats(data.get("formats", [])) - - # Deduplicate tags by namespace to prevent duplicate title:, artist:, etc. - try: - from SYS.metadata import dedup_tags_by_namespace as _dedup - - if _dedup: - tags = _dedup(tags, keep_first=True) - except Exception: - pass # If dedup fails, return tags as-is - - return title, tags, formats, playlist_items - - except subprocess.TimeoutExpired: - log("yt-dlp timeout (>30s)", file=sys.stderr) - return None, [], [], [] + return list(scrape_openlibrary_metadata(olid)) except Exception as e: - log(f"URL scraping error: {e}", file=sys.stderr) - return None, [], [], [] - - -def _extract_url_formats(formats: list) -> List[Tuple[str, str]]: - """Extract best formats from yt-dlp formats list. - - Returns list of (display_label, format_id) tuples. - """ - try: - video_formats = {} # {resolution: format_data} - audio_formats = {} # {quality_label: format_data} - - for fmt in formats: - vcodec = fmt.get("vcodec", "none") - acodec = fmt.get("acodec", "none") - height = fmt.get("height") - ext = fmt.get("ext", "unknown") - format_id = fmt.get("format_id", "") - tbr = fmt.get("tbr", 0) - abr = fmt.get("abr", 0) - - # Video format - if vcodec and vcodec != "none" and height: - if height < 480: - continue - res_key = f"{height}p" - if res_key not in video_formats or tbr > video_formats[res_key].get( - "tbr", - 0): - video_formats[res_key] = { - "label": f"{height}p ({ext})", - "format_id": format_id, - "tbr": tbr, - } - - # Audio-only format - elif acodec and acodec != "none" and (not vcodec or vcodec == "none"): - audio_key = f"audio_{abr}" - if audio_key not in audio_formats or abr > audio_formats[audio_key].get( - "abr", - 0): - audio_formats[audio_key] = { - "label": f"audio ({ext})", - "format_id": format_id, - "abr": abr, - } - - result = [] - - # Add video formats in descending resolution order - for res in sorted(video_formats.keys(), - key=lambda x: int(x.replace("p", "")), - reverse=True): - fmt = video_formats[res] - result.append((fmt["label"], fmt["format_id"])) - - # Add best audio format - if audio_formats: - best_audio = max(audio_formats.values(), key=lambda x: x.get("abr", 0)) - result.append((best_audio["label"], best_audio["format_id"])) - - return result - - except Exception as e: - log(f"Error extracting formats: {e}", file=sys.stderr) + log(f"OpenLibrary scraping error: {e}", file=sys.stderr) return [] @@ -867,14 +224,6 @@ def _scrape_isbn_metadata(isbn: str) -> List[str]: return [] -def _scrape_openlibrary_metadata(olid: str) -> List[str]: - try: - return list(scrape_openlibrary_metadata(olid)) - except Exception as e: - log(f"OpenLibrary scraping error: {e}", file=sys.stderr) - return [] - - def _perform_scraping(tags_list: List[str]) -> List[str]: """Perform scraping based on identifiers in tags. @@ -1044,181 +393,45 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: scrape_url = parsed_args.get("scrape") scrape_requested = scrape_flag_present or scrape_url is not None - # Convenience: `-scrape` with no value defaults to `ytdlp` (store-backed URL scrape). - if scrape_flag_present and (scrape_url is None or str(scrape_url).strip() == ""): - scrape_url = "ytdlp" - scrape_requested = True - - if scrape_requested and (scrape_url is None or str(scrape_url).strip() == ""): - log("-scrape requires a URL or provider name", file=sys.stderr) - return 1 - - # Handle URL or provider scraping mode - if scrape_requested and scrape_url: + # Handle URL or provider scraping mode. + if scrape_requested: import json as json_module - if str(scrape_url).strip().lower() == "ytdlp": - # Scrape metadata from the selected item's URL via yt-dlp (no download), - # then OVERWRITE all existing tags (including title:). - # - # This mode requires a store-backed item (hash + store). - # - # NOTE: We intentionally do not reuse _scrape_url_metadata() here because it - # performs namespace deduplication that would collapse multi-valued tags. - file_hash = normalize_hash(hash_override) or normalize_hash( - get_field(result, - "hash", - None) - ) - store_name = get_field(result, "store", None) - subject_path = ( - get_field(result, - "path", - None) or get_field(result, - "target", - None) - or get_field(result, - "filename", - None) - ) - item_title = ( - get_field(result, - "title", - None) or get_field(result, - "name", - None) - or get_field(result, - "filename", - None) - ) - - # Only run overwrite-apply when the item is store-backed. - # If this is a URL-only PipeObject, fall through to provider mode below. - if (file_hash and store_name and str(file_hash).strip().lower() != "unknown" - and str(store_name).strip().upper() not in {"PATH", - "URL"}): - try: - from Store import Store - - storage = Store(config, suppress_debug=True) - backend = storage[str(store_name)] - except Exception as exc: - log( - f"Failed to resolve store backend '{store_name}': {exc}", - file=sys.stderr - ) - return 1 - - candidate_urls = _resolve_candidate_urls_for_item( - result, - backend, - file_hash, - config - ) - scrape_target = _pick_supported_ytdlp_url(candidate_urls) - if not scrape_target: - log( - "No yt-dlp-supported source URL found for this item (Hydrus /get_files/file URLs are ignored). ", - file=sys.stderr, - ) - log( - "Add the original page URL to the file (e.g. via add-url), then retry get-tag -scrape.", - file=sys.stderr, - ) - return 1 - - ytdlp_provider = get_metadata_provider("ytdlp", config) - if ytdlp_provider is None: - log("yt-dlp metadata provider is unavailable", file=sys.stderr) - return 1 - - try: - tags = [ - str(t) - for t in ytdlp_provider.search_tags(scrape_target, limit=1) - if t is not None - ] - except Exception: - tags = [] - - # Ensure we actually have something to apply. - tags = _dedup_tags_preserve_order(tags) - if not tags: - log("No tags extracted from yt-dlp metadata", file=sys.stderr) - return 1 - - # Full overwrite: delete all existing tags, then add the new set. - try: - existing_tags, _src = backend.get_tag(file_hash, config=config) - except Exception: - existing_tags = [] - try: - if existing_tags: - backend.delete_tag( - file_hash, - list(existing_tags), - config=config - ) - except Exception as exc: - debug(f"[get_tag] ytdlp overwrite: delete_tag failed: {exc}") - try: - backend.add_tag(file_hash, list(tags), config=config) - except Exception as exc: - log(f"Failed to apply yt-dlp tags: {exc}", file=sys.stderr) - return 1 - - # Show updated tags - try: - updated_tags, _src = backend.get_tag(file_hash, config=config) - except Exception: - updated_tags = tags - if not updated_tags: - updated_tags = tags - - _emit_tags_as_table( - tags_list=list(updated_tags), - file_hash=file_hash, - store=str(store_name), - service_name=None, - config=config, - item_title=str(item_title or "ytdlp"), - path=str(subject_path) if subject_path else None, - subject={ - "hash": file_hash, - "store": str(store_name), - "path": str(subject_path) if subject_path else None, - "title": item_title, - "extra": { - "applied_provider": "ytdlp", - "scrape_url": scrape_target - }, - }, - quiet=emit_mode, - ) - return 0 - - if scrape_url.startswith("http://") or scrape_url.startswith("https://"): - # URL scraping (existing behavior) - title, tags, formats, playlist_items = _scrape_url_metadata(scrape_url) - if not tags: - log("No tags extracted from URL", file=sys.stderr) + scrape_target = str(scrape_url or "").strip() if scrape_url is not None else "" + provider = None + if scrape_target.startswith(("http://", "https://")): + provider = get_metadata_provider_for_url(scrape_target, config) + if provider is None: + log("No metadata provider can scrape this URL", file=sys.stderr) return 1 - output = { - "title": title, - "tag": tags, - "formats": [(label, - fmt_id) for label, fmt_id in formats], - "playlist_items": playlist_items, - } - print(json_module.dumps(output, ensure_ascii=False)) + payload = provider.scrape_url_payload(scrape_target) + if not isinstance(payload, dict): + log(f"No metadata extracted from URL via {provider.name}", file=sys.stderr) + return 1 + print(json_module.dumps(payload, ensure_ascii=False)) return 0 - # Provider scraping (e.g., itunes, imdb) - provider = get_metadata_provider(scrape_url, config) + if scrape_target: + provider = get_metadata_provider(scrape_target, config) + else: + provider = get_default_subject_scrape_provider(config) if provider is None: - log(f"Unknown metadata provider: {scrape_url}", file=sys.stderr) + if scrape_target: + log(f"Unknown metadata provider: {scrape_target}", file=sys.stderr) + else: + log("No default metadata provider is available for subject scraping", file=sys.stderr) return 1 + backend = None + if is_store_backed: + try: + from Store import Store + + storage = Store(config, suppress_debug=True) + backend = storage[str(store_name)] + except Exception: + backend = None + # Prefer identifier tags (ISBN/OLID/etc.) when available; fallback to title/filename. # IMPORTANT: do not rely on `result.tag` for this because it can be stale (cached on # the piped PipeObject). Always prefer the current store-backed tags when possible. @@ -1321,17 +534,21 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: except Exception: combined_query = None - # yt-dlp isn't a search provider; it requires a URL. - url_hint: Optional[str] = None + resolved_subject_query: Optional[str] = None try: - url_hint = provider.extract_url_query(result, get_field) + resolved_subject_query = provider.resolve_subject_query( + result, + get_field, + backend=backend, + file_hash=file_hash_for_scrape, + ) except Exception: - url_hint = None + resolved_subject_query = None - query_hint = url_hint or identifier_query or combined_query or title_hint + query_hint = resolved_subject_query or identifier_query or combined_query or title_hint if not query_hint: log( - "No title or identifier available to search for metadata", + f"No query could be resolved for metadata provider '{provider.name}'", file=sys.stderr ) return 1 @@ -1348,7 +565,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: log("No metadata results found", file=sys.stderr) return 1 - # For yt-dlp, emit tags directly (there is no meaningful multi-result selection step). + # Some providers emit tags directly instead of presenting a metadata selection table. emit_direct = False try: emit_direct = bool(provider.emits_direct_tags()) @@ -1359,19 +576,79 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: tags = [str(t) for t in provider.to_tags(items[0]) if t is not None] except Exception: tags = [] + tags = _dedup_tags_preserve_order(tags) if not tags: - log("No tags extracted from yt-dlp metadata", file=sys.stderr) + log(f"No tags extracted from {provider.name} metadata", file=sys.stderr) return 1 + + overwrite_store = False + try: + overwrite_store = bool(is_store_backed and provider.prefers_store_tag_overwrite()) + except Exception: + overwrite_store = False + + if overwrite_store: + if backend is None or not file_hash or not store_name: + log( + f"Failed to resolve store backend for provider '{provider.name}'", + file=sys.stderr, + ) + return 1 + + try: + existing_tags, _src = backend.get_tag(file_hash, config=config) + except Exception: + existing_tags = [] + try: + if existing_tags: + backend.delete_tag(file_hash, list(existing_tags), config=config) + except Exception as exc: + debug(f"[get_tag] {provider.name} overwrite delete_tag failed: {exc}") + try: + backend.add_tag(file_hash, list(tags), config=config) + except Exception as exc: + log(f"Failed to apply {provider.name} tags: {exc}", file=sys.stderr) + return 1 + + try: + updated_tags, _src = backend.get_tag(file_hash, config=config) + except Exception: + updated_tags = tags + if not updated_tags: + updated_tags = tags + + _emit_tags_as_table( + tags_list=list(updated_tags), + file_hash=file_hash, + store=str(store_name), + service_name=None, + config=config, + item_title=str(item_title or provider.name), + path=str(subject_path) if subject_path else None, + subject={ + "hash": file_hash, + "store": str(store_name), + "path": str(subject_path) if subject_path else None, + "title": item_title, + "extra": { + "applied_provider": provider.name, + "scrape_url": str(query_hint), + }, + }, + quiet=emit_mode, + ) + return 0 + _emit_tags_as_table( tags_list=list(tags), file_hash=None, store="url", service_name=None, config=config, - item_title=str(items[0].get("title") or "ytdlp"), + item_title=str(items[0].get("title") or provider.name), path=None, subject={ - "provider": "ytdlp", + "provider": provider.name, "url": str(query_hint) }, quiet=emit_mode, @@ -1402,7 +679,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: None) ) for idx, item in enumerate(items): - tags = _filter_scraped_tags(provider.to_tags(item)) + tags = provider.filter_tags_for_selection(provider.to_tags(item)) add_row_columns( table, [ @@ -1472,13 +749,13 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: ) return 0 - # Apply tags to the store backend (no sidecar writing here). - if str(result_provider).strip().lower() == "ytdlp": - apply_tags = [str(t) for t in result_tags if t is not None] - else: - apply_tags = _filter_scraped_tags( + provider_for_apply = get_metadata_provider(str(result_provider), config) + if provider_for_apply is not None: + apply_tags = provider_for_apply.filter_tags_for_store_apply( [str(t) for t in result_tags if t is not None] ) + else: + apply_tags = _filter_scraped_tags([str(t) for t in result_tags if t is not None]) if not apply_tags: log( "No applicable scraped tags to apply (title:/artist:/source: are skipped)", @@ -1680,11 +957,6 @@ except Exception: "imdb", ] -# Special scrape mode: pull tags from an item's URL via yt-dlp (no download) -if "ytdlp" not in _SCRAPE_CHOICES: - _SCRAPE_CHOICES.append("ytdlp") - _SCRAPE_CHOICES = sorted(_SCRAPE_CHOICES) - class Get_Tag(Cmdlet): """Class-based get-tag cmdlet with self-registration.""" @@ -1715,7 +987,7 @@ class Get_Tag(Cmdlet): name="-scrape", type="string", description= - "Scrape metadata from URL/provider, or use 'ytdlp' to scrape from the item's URL and overwrite tags", + "Scrape metadata from a URL or provider; with no value, use the default subject-scrape provider", required=False, choices=_SCRAPE_CHOICES, ), diff --git a/cmdlet/merge_file.py b/cmdlet/merge_file.py index d76f5c7..5a086dc 100644 --- a/cmdlet/merge_file.py +++ b/cmdlet/merge_file.py @@ -154,47 +154,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: if urls_to_download and len(urls_to_download) >= 2: try: - # Compute a batch hint (audio vs video + single-format id) once. - mode_hint: Optional[str] = None - forced_format: Optional[str] = None - try: - from tool.ytdlp import YtDlpTool, list_formats - - sample_url = urls_to_download[0] - cookiefile = None - try: - cookie_path = YtDlpTool(config).resolve_cookiefile() - if cookie_path is not None and cookie_path.is_file(): - cookiefile = str(cookie_path) - except Exception: - cookiefile = None - - fmts = list_formats( - sample_url, - no_playlist=False, - playlist_items=None, - cookiefile=cookiefile - ) - if isinstance(fmts, list) and fmts: - has_video = False - for f in fmts: - if not isinstance(f, dict): - continue - vcodec = str(f.get("vcodec", "none") or "none").strip().lower() - if vcodec and vcodec != "none": - has_video = True - break - mode_hint = "video" if has_video else "audio" - - if len(fmts) == 1 and isinstance(fmts[0], dict): - fid = str(fmts[0].get("format_id") or "").strip() - if fid: - forced_format = fid - except Exception: - mode_hint = None - forced_format = None - - from cmdlet.download_file import Download_File + from ProviderCore.registry import get_plugin_for_url expanded: List[Dict[str, Any]] = [] downloaded_any = False @@ -207,12 +167,18 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: expanded.append(it) continue - downloaded = Download_File.download_streaming_url_as_pipe_objects( - u, - config, - mode_hint=mode_hint, - ytdl_format_hint=forced_format, - ) + downloaded = [] + try: + plugin = get_plugin_for_url(u, config) + except Exception: + plugin = None + if plugin is not None and hasattr(plugin, "download_url_as_pipe_objects"): + try: + downloaded = plugin.download_url_as_pipe_objects(u) + except TypeError: + downloaded = plugin.download_url_as_pipe_objects(u, output_dir=None) + except Exception: + downloaded = [] if downloaded: expanded.extend(downloaded) downloaded_any = True diff --git a/cmdlet/provider_table.py b/cmdlet/provider_table.py index 6c3435a..b5dba10 100644 --- a/cmdlet/provider_table.py +++ b/cmdlet/provider_table.py @@ -7,7 +7,7 @@ from . import _shared as sh from SYS.logger import log from SYS import pipeline as ctx -from SYS.result_table_adapters import get_provider +from SYS.result_table_adapters import get_plugin from SYS.result_table_renderers import RichRenderer Cmdlet = sh.Cmdlet @@ -16,19 +16,19 @@ parse_cmdlet_args = sh.parse_cmdlet_args CMDLET = Cmdlet( - name="provider-table", - summary="Render a provider's result set and optionally run a follow-up cmdlet using the selected row.", - usage="provider-table -provider [-sample] [-select ] [-run-cmd ]", + name="plugin-table", + summary="Render a plugin's result set and optionally run a follow-up cmdlet using the selected row.", + usage="plugin-table -plugin [-sample] [-select ] [-run-cmd ]", arg=[ - CmdletArg("provider", type="string", description="Provider name to render (default: example)"), - CmdletArg("sample", type="flag", description="Use provider sample/demo items when available."), + CmdletArg("plugin", type="string", description="Plugin name to render (default: example)"), + CmdletArg("sample", type="flag", description="Use plugin sample/demo items when available."), CmdletArg("select", type="int", description="1-based row index to select and use for follow-up command."), CmdletArg("run-cmd", type="string", description="Cmdlet to invoke with the selected row's selector args."), ], detail=[ - "Use a registered provider to build a table and optionally run another cmdlet with selection args.", + "Use a registered plugin to build a table and optionally run another cmdlet with selection args.", "Emits pipeline-friendly dicts enriched with `_selection_args` so you can use @N syntax to select and chain.", - "Example: provider-table -provider example -sample | @1 | add-file -store my_store", + "Example: plugin-table -plugin example -sample | @1 | add-file -store my_store", ], ) @@ -36,15 +36,15 @@ CMDLET = Cmdlet( def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: parsed = parse_cmdlet_args(args, CMDLET) - provider_name = parsed.get("provider") or "example" + plugin_name = parsed.get("plugin") or "example" use_sample = bool(parsed.get("sample", False)) run_cmd = parsed.get("run-cmd") select_raw = parsed.get("select") try: - provider = get_provider(provider_name) + provider = get_plugin(plugin_name) except Exception: - log(f"Unknown provider: {provider_name}", file=sys.stderr) + log(f"Unknown plugin: {plugin_name}", file=sys.stderr) return 1 # Obtain items to feed to the adapter @@ -55,23 +55,23 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: mod = __import__(provider.adapter.__module__, fromlist=["*"]) items = getattr(mod, "SAMPLE_ITEMS", None) if items is None: - log("Provider does not expose SAMPLE_ITEMS; no sample available", file=sys.stderr) + log("Plugin does not expose SAMPLE_ITEMS; no sample available", file=sys.stderr) return 1 except Exception: - log("Failed to load provider sample", file=sys.stderr) + log("Failed to load plugin sample", file=sys.stderr) return 1 else: # Require input for non-sample runs inputs = list(result) if isinstance(result, Iterable) else [] if not inputs: - log("No input provided. Use -sample for demo or pipe provider items in.", file=sys.stderr) + log("No input provided. Use -sample for demo or pipe plugin items in.", file=sys.stderr) return 1 items = inputs try: table = provider.build_table(items) except Exception as exc: - log(f"Provider '{provider.name}' failed: {exc}", file=sys.stderr) + log(f"Plugin '{provider.name}' failed: {exc}", file=sys.stderr) return 1 # Emit rows for downstream pipeline consumption (pipable behavior). diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index 1e0578e..5165cd1 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -15,7 +15,7 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin from SYS.logger import log, debug from SYS.payload_builders import build_file_result_payload, normalize_file_extension -from ProviderCore.registry import get_search_provider, list_search_providers +from ProviderCore.registry import get_search_plugin, list_search_plugins from SYS.rich_display import ( show_provider_config_panel, show_store_config_panel, @@ -169,8 +169,8 @@ class search_file(Cmdlet): def __init__(self) -> None: super().__init__( name="search-file", - summary="Search storage backends (Hydrus) or external providers (via -provider).", - usage="search-file [-query ] [-store BACKEND] [-limit N] [-provider NAME]", + summary="Search storage backends (Hydrus) or external plugins (via -plugin).", + usage="search-file [-query ] [-store BACKEND] [-limit N] [-plugin NAME]", arg=[ CmdletArg( "limit", @@ -179,11 +179,7 @@ class search_file(Cmdlet): ), SharedArgs.STORE, SharedArgs.QUERY, - CmdletArg( - "provider", - type="string", - description="External provider name (e.g., tidal, youtube, soulseek, etc)", - ), + SharedArgs.PLUGIN, CmdletArg( "open", type="integer", @@ -209,10 +205,10 @@ class search_file(Cmdlet): "search-file 'example.com/path' -query 'ext:pdf' # Web: site:example.com filetype:pdf", "search-file -query 'site:example.com filetype:epub history' # Web: site-scoped search", "", - "Provider search (-provider):", - "search-file -provider youtube 'tutorial' # Search YouTube provider", - "search-file -provider alldebrid '*' # List AllDebrid magnets", - "search-file -provider alldebrid -open 123 '*' # Show files for a magnet", + "Plugin search (-plugin):", + "search-file -plugin youtube 'tutorial' # Search YouTube plugin", + "search-file -plugin alldebrid '*' # List AllDebrid magnets", + "search-file -plugin alldebrid -open 123 '*' # Show files for a magnet", ], exec=self.run, ) @@ -1451,10 +1447,10 @@ class search_file(Cmdlet): self._set_storage_display_columns(payload) return payload - def _run_provider_search( + def _run_plugin_search( self, *, - provider_name: str, + plugin_name: str, query: str, limit: int, limit_set: bool, @@ -1463,9 +1459,9 @@ class search_file(Cmdlet): refresh_mode: bool, config: Dict[str, Any], ) -> int: - """Execute external provider search.""" + """Execute external plugin search.""" - if not provider_name or not query: + if not plugin_name or not query: from SYS import pipeline as ctx_mod progress = None if hasattr(ctx_mod, "get_pipeline_state"): @@ -1476,10 +1472,10 @@ class search_file(Cmdlet): except Exception: pass - log("Error: search-file -provider requires both provider and query", file=sys.stderr) + log("Error: search-file -plugin requires both plugin and query", file=sys.stderr) log(f"Usage: {self.usage}", file=sys.stderr) - providers_map = list_search_providers(config) + providers_map = list_search_plugins(config) available = [n for n, a in providers_map.items() if a] unconfigured = [n for n, a in providers_map.items() if not a] @@ -1500,7 +1496,7 @@ class search_file(Cmdlet): if hasattr(ctx_mod, "get_pipeline_state"): progress = ctx_mod.get_pipeline_state().live_progress - provider = get_search_provider(provider_name, config) + provider = get_search_plugin(plugin_name, config) if not provider: if progress: try: @@ -1508,9 +1504,9 @@ class search_file(Cmdlet): except Exception: pass - show_provider_config_panel([provider_name]) + show_provider_config_panel([plugin_name]) - providers_map = list_search_providers(config) + providers_map = list_search_plugins(config) available = [n for n, a in providers_map.items() if a] if available: show_available_providers_panel(available) @@ -1522,7 +1518,7 @@ class search_file(Cmdlet): worker_id, "search-file", title=f"Search: {query}", - description=f"Provider: {provider_name}, Query: {query}", + description=f"Plugin: {plugin_name}, Query: {query}", ) except Exception: pass @@ -1532,7 +1528,7 @@ class search_file(Cmdlet): from SYS.result_table import Table - provider_text = str(provider_name or "").strip() + provider_text = str(plugin_name or "").strip() provider_lower = provider_text.lower() # Dynamic query/filter extraction via provider @@ -1564,9 +1560,9 @@ class search_file(Cmdlet): source_cmd, source_args = provider.get_source_command(args_list) table.set_source_command(source_cmd, source_args) - debug(f"[search-file] Calling {provider_name}.search(filters={search_filters})") + debug(f"[search-file] Calling {plugin_name}.search(filters={search_filters})") results = provider.search(query, limit=limit, filters=search_filters or None) - debug(f"[search-file] {provider_name} -> {len(results or [])} result(s)") + debug(f"[search-file] {plugin_name} -> {len(results or [])} result(s)") # Allow providers to apply provider-specific UX transforms (e.g. auto-expansion) try: @@ -1615,7 +1611,7 @@ class search_file(Cmdlet): # Ensure provider source is present so downstream cmdlets (select) can resolve provider if "source" not in item_dict: - item_dict["source"] = provider_name + item_dict["source"] = plugin_name row_index = len(table.rows) table.add_result(search_result) @@ -1636,7 +1632,7 @@ class search_file(Cmdlet): return 0 except Exception as exc: - log(f"Error searching provider '{provider_name}': {exc}", file=sys.stderr) + log(f"Error searching plugin '{plugin_name}': {exc}", file=sys.stderr) import traceback debug(traceback.format_exc()) @@ -1728,9 +1724,9 @@ class search_file(Cmdlet): f.lower() for f in (flag_registry.get("limit") or {"-limit", "--limit"}) } - provider_flags = { + plugin_flags = { f.lower() - for f in (flag_registry.get("provider") or {"-provider", "--provider"}) + for f in (flag_registry.get("plugin") or {"-plugin", "--plugin"}) } open_flags = { f.lower() @@ -1740,7 +1736,7 @@ class search_file(Cmdlet): # Parse arguments query = "" storage_backend: Optional[str] = None - provider_name: Optional[str] = None + plugin_name: Optional[str] = None open_id: Optional[int] = None limit = 100 limit_set = False @@ -1756,8 +1752,8 @@ class search_file(Cmdlet): query = f"{query} {chunk}".strip() if query else chunk i += 2 continue - if low in provider_flags and i + 1 < len(args_list): - provider_name = args_list[i + 1] + if low in plugin_flags and i + 1 < len(args_list): + plugin_name = args_list[i + 1] i += 2 continue if low in open_flags and i + 1 < len(args_list): @@ -1790,9 +1786,9 @@ class search_file(Cmdlet): query = query.strip() - if provider_name: - return self._run_provider_search( - provider_name=provider_name, + if plugin_name: + return self._run_plugin_search( + plugin_name=plugin_name, query=query, limit=limit, limit_set=limit_set, @@ -1814,56 +1810,6 @@ class search_file(Cmdlet): if store_filter and not storage_backend: storage_backend = store_filter - # If the user accidentally used `-store ` or `store:`, - # prefer to treat it as a provider search (providers like 'alldebrid' are not store backends). - try: - from Store.registry import list_configured_backend_names - providers_map = list_search_providers(config) - configured = list_configured_backend_names(config or {}) - if storage_backend: - matched = None - storage_hint = self._normalize_lookup_target(storage_backend) - if storage_hint: - for p in (providers_map or {}): - if self._normalize_lookup_target(p) == storage_hint: - matched = p - break - if matched and str(storage_backend) not in configured: - log(f"Note: Treating '-store {storage_backend}' as provider search for '{matched}'", file=sys.stderr) - return self._run_provider_search( - provider_name=matched, - query=query, - limit=limit, - limit_set=limit_set, - open_id=open_id, - args_list=args_list, - refresh_mode=refresh_mode, - config=config, - ) - elif store_filter: - matched = None - store_hint = self._normalize_lookup_target(store_filter) - if store_hint: - for p in (providers_map or {}): - if self._normalize_lookup_target(p) == store_hint: - matched = p - break - if matched and str(store_filter) not in configured: - log(f"Note: Treating 'store:{store_filter}' as provider search for '{matched}'", file=sys.stderr) - return self._run_provider_search( - provider_name=matched, - query=query, - limit=limit, - limit_set=limit_set, - open_id=open_id, - args_list=args_list, - refresh_mode=refresh_mode, - config=config, - ) - except Exception: - # Be conservative: if provider detection fails, fall back to store behaviour - pass - hash_query = parse_hash_query(query) web_plan = self._build_web_search_plan( diff --git a/cmdnat/_status_shared.py b/cmdnat/_status_shared.py index b1158ce..3c8ad37 100644 --- a/cmdnat/_status_shared.py +++ b/cmdnat/_status_shared.py @@ -74,34 +74,9 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]: def provider_display_name(key: str) -> str: label = (key or "").strip() - lower = label.lower() - if lower == "openlibrary": - return "OpenLibrary" - if lower == "alldebrid": - return "AllDebrid" - if lower == "youtube": - return "YouTube" return label[:1].upper() + label[1:] if label else "Provider" -def default_provider_ping_targets(provider_key: str) -> list[str]: - provider = (provider_key or "").strip().lower() - if provider == "openlibrary": - return ["https://openlibrary.org"] - if provider == "youtube": - return ["https://www.youtube.com"] - if provider == "bandcamp": - return ["https://bandcamp.com"] - if provider == "libgen": - try: - from Provider.libgen import MIRRORS - - return [str(url).rstrip("/") + "/json.php" for url in (MIRRORS or []) if str(url).strip()] - except ImportError: - return [] - return [] - - def ping_first(urls: list[str]) -> tuple[bool, str]: for url in urls: ok, detail = ping_url(url) @@ -109,4 +84,61 @@ def ping_first(urls: list[str]) -> tuple[bool, str]: return True, detail if urls: return ping_url(urls[0]) - return False, "No ping target" \ No newline at end of file + return False, "No ping target" + + +def collect_plugin_startup_checks(config: dict) -> list[dict[str, Any]]: + provider_cfg = config.get("provider") if isinstance(config, dict) else None + if not isinstance(provider_cfg, dict) or not provider_cfg: + return [] + + try: + from ProviderCore.registry import get_plugin_class + except Exception: + return [] + + checks: list[dict[str, Any]] = [] + for plugin_name in provider_cfg.keys(): + plugin_key = str(plugin_name or "").strip().lower() + if not plugin_key: + continue + + plugin_class = None + try: + plugin_class = get_plugin_class(plugin_key) + except Exception: + plugin_class = None + + if plugin_class is None: + checks.append( + { + "status": "UNKNOWN", + "name": provider_display_name(plugin_key), + "plugin": plugin_key, + "detail": "Not registered", + } + ) + continue + + try: + plugin = plugin_class(config) + summary = plugin.status_summary() + except Exception as exc: + summary = { + "status": "DISABLED", + "name": provider_display_name(plugin_key), + "plugin": plugin_key, + "detail": str(exc), + } + + checks.append( + { + "status": str(summary.get("status") or "UNKNOWN"), + "name": str(summary.get("name") or provider_display_name(plugin_key)), + "plugin": str(summary.get("plugin") or plugin_key), + "detail": str(summary.get("detail") or ""), + "files": summary.get("files"), + } + ) + + return checks \ No newline at end of file diff --git a/cmdnat/matrix.py b/cmdnat/matrix.py index 7ed64c3..5bd7ffb 100644 --- a/cmdnat/matrix.py +++ b/cmdnat/matrix.py @@ -15,6 +15,7 @@ from SYS.result_table import Table from SYS.item_accessors import get_sha256_hex from SYS.utils import extract_hydrus_hash_from_url from SYS import pipeline as ctx +from ProviderCore.registry import get_plugin, get_plugin_for_url from cmdnat._parsing import ( extract_arg_value, extract_piped_value as _extract_piped_value, @@ -29,6 +30,29 @@ _MATRIX_MENU_STATE_KEY = "matrix_menu_state" _MATRIX_SELECTED_SETTING_KEY_KEY = "matrix_selected_setting_key" +def _get_matrix_provider(config: Dict[str, Any]) -> Any: + provider = get_plugin("matrix", config) + if provider is None: + raise RuntimeError("Matrix plugin is not registered") + return provider + + +def _resolve_plugin_url(url: str, config: Dict[str, Any]) -> str: + target = str(url or "").strip() + if not target: + return target + + provider = get_plugin_for_url(target, config) + if provider is None: + return target + + try: + resolved = provider.resolve_url(target) + except Exception: + return target + return str(resolved or target) + + def _extract_set_value_arg(args: Sequence[str]) -> Optional[str]: """Extract the value from -set-value flag.""" return extract_arg_value(args, flags={"-set-value"}) @@ -212,35 +236,11 @@ def _resolve_room_identifier(value: str, config: Dict[str, Any]) -> Optional[str conf_ids = _parse_config_room_filter_ids(config) if conf_ids: # Attempt to fetch names for the configured IDs - try: - from Provider.matrix import Matrix - # Avoid __init__ network failures by requiring homeserver+token to exist - block = config.get("provider", {}).get("matrix", {}) - if block and block.get("homeserver") and block.get("access_token"): - try: - m = Matrix(config) - rooms = m.list_rooms(room_ids=conf_ids) - for room in rooms or []: - name = str(room.get("name") or "").strip() - rid = str(room.get("room_id") or "").strip() - if name and name.lower() == cand.lower(): - return rid - if name and cand.lower() in name.lower(): - return rid - except Exception: - # Best-effort; fallback below - pass - except Exception: - pass - - # Last resort: attempt to ask the server for matching rooms (if possible) - try: - from Provider.matrix import Matrix block = config.get("provider", {}).get("matrix", {}) if block and block.get("homeserver") and block.get("access_token"): try: - m = Matrix(config) - rooms = m.list_rooms() + m = _get_matrix_provider(config) + rooms = m.list_rooms(room_ids=conf_ids) for room in rooms or []: name = str(room.get("name") or "").strip() rid = str(room.get("room_id") or "").strip() @@ -250,8 +250,22 @@ def _resolve_room_identifier(value: str, config: Dict[str, Any]) -> Optional[str return rid except Exception: pass - except Exception: - pass + + # Last resort: attempt to ask the server for matching rooms (if possible) + block = config.get("provider", {}).get("matrix", {}) + if block and block.get("homeserver") and block.get("access_token"): + try: + m = _get_matrix_provider(config) + rooms = m.list_rooms() + for room in rooms or []: + name = str(room.get("name") or "").strip() + rid = str(room.get("room_id") or "").strip() + if name and name.lower() == cand.lower(): + return rid + if name and cand.lower() in name.lower(): + return rid + except Exception: + pass return None except Exception: @@ -270,10 +284,8 @@ def _send_pending_to_rooms(config: Dict[str, Any], room_ids: List[str], args: Se log("No pending items to upload (use: @N | .matrix)", file=sys.stderr) return 1 - from Provider.matrix import Matrix - try: - provider = Matrix(config) + provider = _get_matrix_provider(config) except Exception as exc: log(f"Matrix not available: {exc}", file=sys.stderr) return 1 @@ -585,35 +597,6 @@ def _maybe_download_hydrus_file(item: Any, return None -def _maybe_unlock_alldebrid_url(url: str, config: Dict[str, Any]) -> str: - try: - parsed = urlparse(url) - host = (parsed.netloc or "").lower() - if host != "alldebrid.com": - return url - if not (parsed.path or "").startswith("/f/"): - return url - - try: - from Provider.alldebrid import _get_debrid_api_key # type: ignore - - api_key = _get_debrid_api_key(config or {}) - except Exception: - api_key = None - if not api_key: - return url - - from API.alldebrid import AllDebridClient - - client = AllDebridClient(str(api_key)) - unlocked = client.unlock_link(url) - if isinstance(unlocked, str) and unlocked.strip(): - return unlocked.strip() - except Exception: - pass - return url - - def _resolve_upload_path(item: Any, config: Dict[str, Any]) -> Optional[str]: """Resolve a usable local file path for uploading. @@ -645,7 +628,7 @@ def _resolve_upload_path(item: Any, config: Dict[str, Any]) -> Optional[str]: return None # Best-effort: unlock AllDebrid file links (they require auth and aren't directly downloadable). - url = _maybe_unlock_alldebrid_url(url, config) + url = _resolve_plugin_url(url, config) try: from API.HTTP import _download_direct_file @@ -851,10 +834,8 @@ def _handle_settings_edit(result: Any, args: Sequence[str], config: Dict[str, An def _handle_settings_test(config: Dict[str, Any]) -> int: """Test Matrix credentials and prompt for default rooms upon success.""" - from Provider.matrix import Matrix - try: - provider = Matrix(config) + provider = _get_matrix_provider(config) except Exception as exc: log(f"Matrix test failed: {exc}", file=sys.stderr) return 1 @@ -863,13 +844,11 @@ def _handle_settings_test(config: Dict[str, Any]) -> int: return _show_default_room_picker(config, provider=provider) -def _show_default_room_picker(config: Dict[str, Any], *, provider: Optional["Matrix"] = None) -> int: +def _show_default_room_picker(config: Dict[str, Any], *, provider: Optional[Any] = None) -> int: """Display joined rooms so the user can select defaults for sharing.""" - from Provider.matrix import Matrix - try: if provider is None: - provider = Matrix(config) + provider = _get_matrix_provider(config) except Exception as exc: log(f"Matrix not available: {exc}", file=sys.stderr) return 1 @@ -977,10 +956,8 @@ def _handle_settings_rooms(result: Any, args: Sequence[str], config: Dict[str, A def _show_rooms_table(config: Dict[str, Any]) -> int: """Display rooms (refactored original behavior).""" - from Provider.matrix import Matrix - try: - provider = Matrix(config) + provider = _get_matrix_provider(config) except Exception as exc: log(f"Matrix not available: {exc}", file=sys.stderr) return 1 @@ -1121,10 +1098,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: log("No pending items to upload (use: @N | .matrix)", file=sys.stderr) return 1 - from Provider.matrix import Matrix - try: - provider = Matrix(config) + provider = _get_matrix_provider(config) except Exception as exc: log(f"Matrix not available: {exc}", file=sys.stderr) return 1 diff --git a/cmdnat/pipe.py b/cmdnat/pipe.py index 6d8c8b8..2fe2023 100644 --- a/cmdnat/pipe.py +++ b/cmdnat/pipe.py @@ -10,17 +10,15 @@ from datetime import datetime, timedelta from urllib.parse import urlparse, parse_qs from pathlib import Path from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args -from Provider.tidal_manifest import resolve_tidal_manifest_path +from ProviderCore.registry import get_plugin_for_url from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream from SYS.result_table import Table from MPV.mpv_ipc import MPV from SYS import pipeline as ctx from SYS.models import PipeObject -from SYS.config import get_hydrus_access_key, get_hydrus_url +from SYS.config import get_hydrus_access_key, get_hydrus_url, resolve_cookies_path -_ALLDEBRID_UNLOCK_CACHE: Dict[str, - str] = {} _NOTES_PREFETCH_INFLIGHT: set[str] = set() _NOTES_PREFETCH_LOCK = threading.Lock() _PLAYLIST_STORE_CACHE: Optional[Dict[str, Any]] = None @@ -478,73 +476,85 @@ def _try_enable_mpv_file_logging(mpv_log_path: str, *, attempts: int = 3) -> boo return bool(ok) -def _get_alldebrid_api_key(config: Optional[Dict[str, Any]]) -> Optional[str]: +def _iter_plugin_targets(item: Any) -> List[str]: + values: List[str] = [] + seen: set[str] = set() + + def _add(candidate: Any) -> None: + text = str(candidate or "").strip() + if not text or text in seen: + return + seen.add(text) + values.append(text) + try: - if not isinstance(config, dict): - return None - provider_cfg = config.get("provider") - if not isinstance(provider_cfg, dict): - return None - ad_cfg = provider_cfg.get("alldebrid") - if not isinstance(ad_cfg, dict): - return None - key = ad_cfg.get("api_key") - if not isinstance(key, str): - return None - key = key.strip() - return key or None + if isinstance(item, dict): + _add(item.get("path")) + _add(item.get("url")) + _add(item.get("source_url")) + _add(item.get("target")) + metadata = item.get("full_metadata") or item.get("metadata") + else: + _add(getattr(item, "path", None)) + _add(getattr(item, "url", None)) + _add(getattr(item, "source_url", None)) + _add(getattr(item, "target", None)) + metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None) + if isinstance(metadata, dict): + _add(metadata.get("url")) + _add(metadata.get("webpage_url")) + _add(metadata.get("source_url")) + extra = item.get("extra") if isinstance(item, dict) else getattr(item, "extra", None) + if isinstance(extra, dict): + _add(extra.get("url")) + _add(extra.get("source_url")) except Exception: - return None + return values + + return values -def _is_alldebrid_protected_url(url: str) -> bool: +def _resolve_plugin_url(url: str, config: Optional[Dict[str, Any]]) -> str: + target = str(url or "").strip() + if not target: + return target + try: - if not isinstance(url, str): - return False - u = url.strip() - if not u.startswith(("http://", "https://")): - return False - p = urlparse(u) - host = (p.netloc or "").lower() - path = p.path or "" - # AllDebrid file page links (require auth; not directly streamable by mpv) - return host == "alldebrid.com" and path.startswith("/f/") + plugin = get_plugin_for_url(target, config or {}) except Exception: - return False - - -def _maybe_unlock_alldebrid_url(url: str, config: Optional[Dict[str, Any]]) -> str: - """Convert AllDebrid protected file URLs into direct streamable links. - - When AllDebrid returns `https://alldebrid.com/f/...`, that URL typically requires - authentication. MPV cannot access it without credentials. We transparently call - the AllDebrid API `link/unlock` (using the configured API key) to obtain a direct - URL that MPV can stream. - """ - if not _is_alldebrid_protected_url(url): - return url - - cached = _ALLDEBRID_UNLOCK_CACHE.get(url) - if isinstance(cached, str) and cached: - return cached - - api_key = _get_alldebrid_api_key(config) - if not api_key: - return url + plugin = None + if plugin is None: + return target try: - from API.alldebrid import AllDebridClient + resolved = plugin.resolve_url(target) + except Exception as exc: + debug(f"Plugin URL resolution failed for {target}: {exc}", file=sys.stderr) + return target - client = AllDebridClient(api_key) - unlocked = client.unlock_link(url) - if isinstance(unlocked, str) and unlocked.strip(): - unlocked = unlocked.strip() - _ALLDEBRID_UNLOCK_CACHE[url] = unlocked - return unlocked - except Exception as e: - debug(f"AllDebrid unlock failed for MPV target: {e}", file=sys.stderr) + return str(resolved or target) - return url + +def _resolve_plugin_playback_path(item: Any, config: Optional[Dict[str, Any]]) -> Optional[str]: + for candidate in _iter_plugin_targets(item): + try: + plugin = get_plugin_for_url(candidate, config or {}) + except Exception: + plugin = None + if plugin is None: + continue + + try: + resolved = plugin.resolve_playback_path(item) + except Exception as exc: + debug(f"Plugin playback-path resolution failed for {candidate}: {exc}", file=sys.stderr) + continue + + text = str(resolved or "").strip() + if text: + return text + + return None def _ensure_lyric_overlay(mpv: MPV) -> None: @@ -1078,9 +1088,7 @@ def _ensure_ytdl_cookies(config: Optional[Dict[str, Any]] = None) -> None: cookies_path = None try: - from tool.ytdlp import YtDlpTool - - cookiefile = YtDlpTool(config or {}).resolve_cookiefile() + cookiefile = resolve_cookies_path(config or {}) if cookiefile is not None: cookies_path = str(cookiefile) except Exception: @@ -1326,7 +1334,7 @@ def _get_playable_path( "none"}: path = None - manifest_path = resolve_tidal_manifest_path(item) + manifest_path = _resolve_plugin_playback_path(item, config) if manifest_path: path = manifest_path else: @@ -1542,7 +1550,7 @@ def _queue_items( # If the target is an AllDebrid protected file URL, unlock it to a direct link for MPV. try: if isinstance(target, str): - target = _maybe_unlock_alldebrid_url(target, config) + target = _resolve_plugin_url(target, config) except Exception: pass @@ -2591,7 +2599,7 @@ def _start_mpv( try: needs_mpd_whitelist = False for it in items or []: - mpd = resolve_tidal_manifest_path(it) + mpd = _resolve_plugin_playback_path(it, config) candidate = mpd if not candidate: if isinstance(it, dict): diff --git a/cmdnat/status.py b/cmdnat/status.py index be8195d..18d9cab 100644 --- a/cmdnat/status.py +++ b/cmdnat/status.py @@ -4,18 +4,14 @@ import shutil from typing import Any, Dict, List from SYS.cmdlet_spec import Cmdlet +from SYS.config import resolve_cookies_path from SYS import pipeline as ctx from SYS.result_table import Table from SYS.logger import set_debug, debug from cmdnat._status_shared import ( add_startup_check as _add_startup_check, - default_provider_ping_targets as _default_provider_ping_targets, - has_provider as _has_provider, + collect_plugin_startup_checks as _collect_plugin_startup_checks, has_store_subtype as _has_store_subtype, - has_tool as _has_tool, - ping_first as _ping_first, - ping_url as _ping_url, - provider_display_name as _provider_display_name, ) CMDLET = Cmdlet( @@ -95,82 +91,19 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int: detail = f"{uval} - {err or 'Unavailable'}" _add_startup_check(startup_table, status, nkey, store="hydrusnetwork", files=files, detail=detail) - # Providers - pcfg = config.get("provider", {}) - if isinstance(pcfg, dict) and pcfg: - from ProviderCore.registry import list_providers, list_search_providers, list_file_providers - from Provider.metadata_provider import list_metadata_providers - - p_avail = list_providers(config) or {} - s_avail = list_search_providers(config) or {} - f_avail = list_file_providers(config) or {} - m_avail = list_metadata_providers(config) or {} - debug(f"Provider registries: providers={list(p_avail.keys())}, search={list(s_avail.keys())}, file={list(f_avail.keys())}, metadata={list(m_avail.keys())}") - - already = {"matrix"} - for pname in pcfg.keys(): - prov = str(pname).lower() - if prov in already: continue - display = _provider_display_name(prov) - - if prov == "alldebrid": - try: - from Provider.alldebrid import _get_debrid_api_key - from API.alldebrid import AllDebridClient - api_key = _get_debrid_api_key(config) - debug(f"AllDebrid configured: api_key_present={bool(api_key)}") - if not api_key: - _add_startup_check(startup_table, "DISABLED", display, provider=prov, detail="Not configured") - else: - client = AllDebridClient(api_key) - _add_startup_check(startup_table, "ENABLED", display, provider=prov, detail=getattr(client, "base_url", "Connected")) - debug(f"AllDebrid client connected: base_url={getattr(client, 'base_url', 'unknown')}") - except Exception as exc: - _add_startup_check(startup_table, "DISABLED", display, provider=prov, detail=str(exc)) - debug(f"AllDebrid check failed: {exc}") - already.add(prov) - continue - - is_known = prov in p_avail or prov in s_avail or prov in f_avail or prov in m_avail - if not is_known: - _add_startup_check(startup_table, "UNKNOWN", display, provider=prov, detail="Not registered") - debug(f"Provider {prov} not registered") - else: - ok_val = p_avail.get(prov) or s_avail.get(prov) or f_avail.get(prov) or m_avail.get(prov) - detail = "Configured" if ok_val else "Not configured" - ping_targets = _default_provider_ping_targets(prov) - if ping_targets: - debug(f"Provider {prov} ping targets: {ping_targets}") - pok, pdet = _ping_first(ping_targets) - debug(f"Provider {prov} ping result: ok={pok}, detail={pdet}") - detail = pdet if ok_val else f"{detail} | {pdet}" - _add_startup_check(startup_table, "ENABLED" if ok_val else "DISABLED", display, provider=prov, detail=detail) - already.add(prov) - - # Matrix - if _has_provider(config, "matrix"): - try: - from Provider.matrix import Matrix - m_prov = Matrix(config) - mcfg = config.get("provider", {}).get("matrix", {}) - hs = str(mcfg.get("homeserver") or "").strip() - rid = str(mcfg.get("room_id") or "").strip() - detail = f"{hs} room:{rid}" - valid = False - try: - valid = bool(m_prov.validate()) - except Exception as exc: - debug(f"Matrix validate failed: {exc}") - _add_startup_check(startup_table, "ENABLED" if valid else "DISABLED", "Matrix", provider="matrix", detail=detail) - debug(f"Matrix check: homeserver={hs}, room_id={rid}, validate={valid}") - except Exception as exc: - _add_startup_check(startup_table, "DISABLED", "Matrix", provider="matrix", detail=str(exc)) - debug(f"Matrix instantiation failed: {exc}") + for check in _collect_plugin_startup_checks(config): + _add_startup_check( + startup_table, + str(check.get("status") or "UNKNOWN"), + str(check.get("name") or "Plugin"), + provider=str(check.get("plugin") or ""), + files=check.get("files"), + detail=str(check.get("detail") or ""), + ) # Cookies try: - from tool.ytdlp import YtDlpTool - cf = YtDlpTool(config).resolve_cookiefile() + cf = resolve_cookies_path(config) _add_startup_check(startup_table, "FOUND" if cf else "MISSING", "Cookies", detail=str(cf) if cf else "Not found") debug(f"Cookies: resolved cookiefile={cf}") except Exception as exc: diff --git a/cmdnat/telegram.py b/cmdnat/telegram.py index 15c89fd..dc3cb94 100644 --- a/cmdnat/telegram.py +++ b/cmdnat/telegram.py @@ -8,10 +8,18 @@ from SYS.cmdlet_spec import Cmdlet, CmdletArg from SYS.logger import log from SYS.result_table import Table from SYS import pipeline as ctx +from ProviderCore.registry import get_plugin from cmdnat._parsing import has_flag as _has_flag, normalize_to_list as _normalize_to_list _TELEGRAM_PENDING_ITEMS_KEY = "telegram_pending_items" + +def _get_telegram_provider(config: Dict[str, Any]) -> Any: + provider = get_plugin("telegram", config) + if provider is None: + raise RuntimeError("Telegram plugin is not registered") + return provider + def _extract_chat_id(chat_obj: Any) -> Optional[int]: try: if isinstance(chat_obj, dict): @@ -119,10 +127,8 @@ def _extract_file_path(item: Any) -> Optional[str]: def _run(_result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: - from Provider.telegram import Telegram - try: - provider = Telegram(config) + provider = _get_telegram_provider(config) except Exception as exc: log(f"Telegram not available: {exc}", file=sys.stderr) return 1 diff --git a/docs/provider_authoring.md b/docs/provider_authoring.md index d57f286..f8e2f46 100644 --- a/docs/provider_authoring.md +++ b/docs/provider_authoring.md @@ -1,13 +1,13 @@ -# Provider authoring: ResultTable & provider adapters ✅ +# Plugin authoring: ResultTable & plugin adapters -This short guide explains how to write providers that integrate with the *strict* ResultTable API: adapters must yield `ResultModel` instances and providers register via `SYS.result_table_adapters.register_provider` with a column specification and a `selection_fn`. +This short guide explains how to write plugins that integrate with the *strict* ResultTable API: adapters must yield `ResultModel` instances and plugins register via `SYS.result_table_adapters.register_plugin` with a column specification and a `selection_fn`. --- ## Quick summary -- Providers register a *provider adapter* (callable that yields `ResultModel`). -- Providers must also provide `columns` (static list or factory) and a `selection_fn` that returns CLI args for a selected row. +- Plugins register a *plugin adapter* (callable that yields `ResultModel`). +- Plugins must also provide `columns` (static list or factory) and a `selection_fn` that returns CLI args for a selected row. - For simple HTML table/list scraping, prefer `TableProviderMixin` from `SYS.provider_helpers` to fetch and extract rows using `SYS.html_table.extract_records`. ## Runtime dependency policy @@ -21,11 +21,11 @@ This short guide explains how to write providers that integrate with the *strict ## Minimal provider template (copy/paste) ```py -# Provider/my_provider.py +# plugins/my_plugin.py from typing import Any, Dict, Iterable, List from SYS.result_table_api import ResultModel, ColumnSpec, title_column, metadata_column -from SYS.result_table_adapters import register_provider +from SYS.result_table_adapters import register_plugin # Example adapter: convert provider-specific items into ResultModel instances SAMPLE_ITEMS = [ @@ -59,8 +59,8 @@ def selection_fn(row: ResultModel) -> List[str]: return ["-path", row.path] return ["-title", row.title or ""] -# Register provider (done at import time) -register_provider("myprovider", adapter, columns=columns_factory, selection_fn=selection_fn) +# Register plugin (done at import time) +register_plugin("myprovider", adapter, columns=columns_factory, selection_fn=selection_fn) ``` --- @@ -84,7 +84,7 @@ class MyTableProvider(TableProviderMixin, Provider): return self.search_table_from_url(url, limit=limit) ``` -`TableProviderMixin.search_table_from_url` returns `ProviderCore.base.SearchResult` entries. If you want to integrate this provider with the strict `ResultTable` registry, add a small adapter that converts `SearchResult` -> `ResultModel` and register it using `register_provider` (see `Provider/vimm.py` for a real example). +`TableProviderMixin.search_table_from_url` returns `ProviderCore.base.SearchResult` entries. If you want to integrate this plugin with the strict `ResultTable` registry, add a small adapter that converts `SearchResult` -> `ResultModel` and register it using `register_plugin` (see `Provider/vimm.py` for a real example). --- @@ -93,7 +93,7 @@ class MyTableProvider(TableProviderMixin, Provider): - `columns` may be a static `List[ColumnSpec]` or a factory `def cols(rows: List[ResultModel]) -> List[ColumnSpec]` that inspects sample rows. - `selection_fn` must accept a `ResultModel` and return a `List[str]` representing CLI args (e.g., `['-path', row.path]`). These args are used by `select` and `@N` expansion. - **Tip:** for providers that produce downloadable file rows prefer returning explicit URL args (e.g., `['-url', row.path]`) so the selected URL is clearly identified by downstream downloaders and to avoid ambiguous parsing when provider hints (like `-provider`) are present. + **Tip:** for plugins that produce downloadable file rows prefer returning explicit URL args (e.g., `['-url', row.path]`) so the selected URL is clearly identified by downstream downloaders and to avoid ambiguous parsing when plugin hints (like `-plugin`) are present. - Ensure your `ResultModel.source` is set (either in the model or rely on the provider name set by `serialize_row`). --- @@ -107,7 +107,7 @@ class MyTableProvider(TableProviderMixin, Provider): ## Testing & examples - Write `tests/test_provider_.py` that imports your provider and verifies `provider.build_table(...)` produces a `ResultTable` (has `.rows` and `.columns`) and that `serialize_rows()` yields dicts with `_selection_args`, `_selection_action` when applicable, and `source`. -- When you need to guarantee a specific CLI stage sequence (e.g., `download-file -url -provider `), call `table.set_row_selection_action(row_index, tokens)` so the serialized payload emits `_selection_action` and the CLI can run the row exactly as intended. +- When you need to guarantee a specific CLI stage sequence (e.g., `download-file -url -plugin `), call `table.set_row_selection_action(row_index, tokens)` so the serialized payload emits `_selection_action` and the CLI can run the row exactly as intended. - For table providers you can test `search_table_from_url` using a local HTML fixture or by mocking `HTTPClient` to return a small sample page. - If you rely on pandas, add a test that monkeypatches `sys.modules['pandas']` to a simple shim to validate the pandas path. @@ -119,7 +119,7 @@ from Provider import example_provider def test_example_provider_registration(): - provider = get_provider("example") + plugin = get_plugin("example") rows = list(provider.adapter(example_provider.SAMPLE_ITEMS)) assert rows and rows[0].title cols = provider.get_columns(rows) diff --git a/docs/provider_guide.md b/docs/provider_guide.md index 3e5b0e7..1de6ed1 100644 --- a/docs/provider_guide.md +++ b/docs/provider_guide.md @@ -1,23 +1,23 @@ -# Provider Development Guide +# Plugin Development Guide ## 🎯 Purpose -This guide describes how to write, test, and register a provider so the application can discover and use it as a pluggable component. +This guide describes how to write, test, and register a plugin so the application can discover and use it as a pluggable component. -> Keep provider code small, focused, and well-tested. Use existing providers as examples. +> Keep plugin code small, focused, and well-tested. Built-in plugins live in `Provider/` and external drop-in plugins live under `plugins/`. --- -## 🔧 Anatomy of a Provider -A provider is a Python class that extends `ProviderCore.base.Provider` and implements a few key methods and attributes. +## 🔧 Anatomy of a Plugin +A plugin is a Python class that extends `ProviderCore.base.Provider` and implements a few key methods and attributes. Minimum expectations: -- `class MyProvider(Provider):` — subclass the base provider +- `class MyPlugin(Provider):` — subclass the base plugin class - `URL` / `URL_DOMAINS` or `url_patterns()` — to let the registry route URLs -- `validate(self) -> bool` — return True when provider is configured and usable +- `validate(self) -> bool` — return True when the plugin is configured and usable - `search(self, query, limit=50, filters=None, **kwargs)` — return a list of `SearchResult` Optional but common: -- `download(self, result: SearchResult, output_dir: Path) -> Optional[Path]` — download a provider result +- `download(self, result: SearchResult, output_dir: Path) -> Optional[Path]` — download a plugin result - `selector(self, selected_items, *, ctx, stage_is_last=True, **kwargs) -> bool` — handle `@N` selections - `download_url(self, url, output_dir, progress_cb=None)` — direct URL-handling helper @@ -71,8 +71,8 @@ class HelloProvider(Provider): --- ## ⬇️ Implementing download() and download_url() -- Prefer provider `download(self, result, output_dir)` for piped provider items. -- For provider-provided URLs, implement `download_url` to allow `download-file` to route downloads through providers. +- Prefer plugin `download(self, result, output_dir)` for piped plugin items. +- For plugin-provided URLs, implement `download_url` to allow `download-file` to route downloads through plugins. - Use the repo `_download_direct_file` helper for HTTP downloads when possible. Example download(): @@ -90,12 +90,12 @@ def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: --- ## 🧭 URL routing -Providers can declare: +Plugins can declare: - `URL = ("magnet:",)` or similar prefix list - `URL_DOMAINS = ("example.com",)` to match hosts - Or override `@classmethod def url_patterns(cls):` to combine static and dynamic patterns -The registry uses these to match `download-file ` or to pick which provider should handle the URL. +The registry uses these to match `download-file ` or to pick which plugin should handle the URL. --- @@ -106,8 +106,8 @@ The registry uses these to match `download-file ` or to pick which provider --- -## 🧪 Testing providers -- Keep tests small and local. Create `tests/test_provider_.py`. +## 🧪 Testing plugins +- Keep tests small and local. Create `tests/test_provider_.py` or another tracked test target. - Test `search()` with mock HTTP responses (use `requests-mock` or similar). - Test `download()` using a temp directory and a small file server or by mocking `_download_direct_file`. - Test `selector()` by constructing a fake result and `ctx` object. @@ -125,10 +125,9 @@ pytest -q --- ## 📦 Registration & packaging -- Add your provider module under `Provider/` and ensure it is imported by module package initialization. Common approach: - - Place file `Provider/myprovider.py` - - Ensure `Provider/__init__.py` imports the module (or the registry auto-discovers by package import) -- If the project has a central provider registry, add lookup helpers there (e.g., `ProviderCore/registry.py`). Usually providers register themselves at import time. +- Built-in plugins live under `Provider/` and are auto-discovered from that package. +- External user plugins can be dropped into `plugins/` or any directory listed in `MM_PLUGIN_PATH` / `MEDEIA_PLUGIN_PATH`. +- Plugin authors should import from `ProviderCore.*`. --- @@ -147,19 +146,19 @@ pytest -q - [ ] Provide `URL` / `URL_DOMAINS` or `url_patterns()` for routing - [ ] Add `download()` or `download_url()` for piped/passed URL downloads - [ ] Add tests under `tests/` -- [ ] Add module to `Provider/` package and ensure import/registration +- [ ] Add the plugin module to `Provider/` for built-ins, or drop it into `plugins/` for plug-and-play user installs --- ## 🔗 Further reading -- See existing providers in `Provider/` for patterns and edge cases. +- See existing built-in plugins in `Provider/` for patterns and edge cases. - Check `API/` helpers for HTTP and debrid clients. --- If you'd like, I can: -- Add an example provider file under `Provider/` as a template (see `Provider/hello_provider.py`), and +- Add an example plugin file under `Provider/` as a template (see `Provider/hello_provider.py`), and - Create unit tests for it (see `tests/test_provider_hello.py`). I have added a minimal example provider and tests in this repository; use them as a starting point for new providers. diff --git a/docs/result_table.md b/docs/result_table.md index d31310f..91fa0cd 100644 --- a/docs/result_table.md +++ b/docs/result_table.md @@ -40,7 +40,7 @@ from SYS.result_table import ResultTable table = ResultTable("Provider: X result").set_preserve_order(True) table.set_table("provider_name") table.set_table_metadata({"provider":"provider_name","view":"folders"}) -table.set_source_command("search-file", ["-provider","provider_name","query"]) +table.set_source_command("search-file", ["-plugin","provider_name","query"]) for r in results: table.add_result(r) # r can be a SearchResult, dict, or PipeObject @@ -82,13 +82,13 @@ Example commands: ``` # List magnets in your account -search-file -provider alldebrid "*" +search-file -plugin alldebrid "*" # Open magnet id 123 and list its files -search-file -provider alldebrid -open 123 "*" +search-file -plugin alldebrid -open 123 "*" # Or expand via @ selection (selector handles drilling): -search-file -provider alldebrid "*" +search-file -plugin alldebrid "*" @3 # selector will open the magnet referenced by row #3 and show the file table ``` @@ -147,7 +147,7 @@ Selection & download flows ``` # Expand magnet and add first file to local directory -search-file -provider alldebrid "*" +search-file -plugin alldebrid "*" @3 # view files @1 | add-file -path C:\mydir ``` @@ -167,7 +167,7 @@ Example usage: ``` # Search for an artist -search-file -provider bandcamp "artist:radiohead" +search-file -plugin bandcamp "artist:radiohead" # Select an artist row to expand into releases @1 diff --git a/docs/result_table_selector.md b/docs/result_table_selector.md index 87180e1..3c51e8f 100644 --- a/docs/result_table_selector.md +++ b/docs/result_table_selector.md @@ -1,22 +1,22 @@ -Selector & provider-table usage +Selector & plugin-table usage -This project provides a small provider/table/selector flow that allows providers +This project provides a small plugin/table/selector flow that allows plugins and cmdlets to interact via a simple, pipable API. Key ideas -- `provider-table` renders a provider result set and *emits* pipeline-friendly dicts for each row. Each emitted item includes `_selection_args`, a list of args the provider suggests for selecting that row (e.g., `['-path', '/tmp/file']`). +- `plugin-table` renders a plugin result set and *emits* pipeline-friendly dicts for each row. Each emitted item includes `_selection_args`, a list of args the plugin suggests for selecting that row (e.g., `['-path', '/tmp/file']`). - Use the `@N` syntax to select an item from a table and chain it to the next cmdlet. Example: - provider-table -provider example -sample | @1 | add-file -store default + plugin-table -plugin example -sample | @1 | add-file -store default -What providers must implement +What plugins must implement - An adapter that yields `ResultModel` objects (breaking API). - Optionally supply a `columns` factory and `selection_fn` (see `Provider/example_provider.py`). Implementation notes -- `provider-table` emits dicts like `{ 'title': ..., 'path': ..., 'metadata': ..., '_selection_args': [...] }`. -- Selection syntax (`@1`) will prefer `_selection_args` if present; otherwise it will fall back to provider selection logic or sensible defaults (`-path` or `-title`). +- `plugin-table` emits dicts like `{ 'title': ..., 'path': ..., 'metadata': ..., '_selection_args': [...] }`. +- Selection syntax (`@1`) will prefer `_selection_args` if present; otherwise it will fall back to plugin selection logic or sensible defaults (`-path` or `-title`). This design keeps the selector-focused UX small and predictable while enabling full cmdlet interoperability via piping and `-run-cmd`. diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..3ef179d --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,39 @@ +# External Plugins + +Drop user plugins in this folder to make them available to the app without editing the built-in `Provider/` package. + +Supported discovery paths: +- `plugins/` in the repo root +- `plugins/` in the current working directory +- Any directory listed in `MM_PLUGIN_PATH` +- Any directory listed in `MEDEIA_PLUGIN_PATH` + +Plugin module rules: +- A plugin can be a single `.py` file or a package directory with `__init__.py`. +- Define a class that inherits from `ProviderCore.base.Provider`. +- Give it a stable name using `PLUGIN_NAME` or the class name. + +Example skeleton: + +```python +from ProviderCore.base import Provider, SearchResult + + +class MyPlugin(Provider): + PLUGIN_NAME = "myplugin" + URL_DOMAINS = ("example.com",) + + def search(self, query, limit=50, filters=None, **kwargs): + text = str(query or "").strip() + if not text: + return [] + return [ + SearchResult( + table="myplugin", + title=f"Result for {text}", + path=f"https://example.com/{text}", + ) + ] +``` + +Built-in plugins still live in `Provider/`. \ No newline at end of file diff --git a/readme.md b/readme.md index 1949aaa..f4f78f2 100644 --- a/readme.md +++ b/readme.md @@ -33,9 +33,9 @@ Medios-Macina is a API driven file media manager and virtual toolbox capable of
  • no opening of folders neccessary! You can add multiple tags to a file and use the search engine to immediately find and retrieve that file your looking for
  • Flexible syntax structure: chain commands with `|` and select options from tables with `@N`.
  • Multiple file stores: *HYDRUSNETWORK* -- **Provider plugin integration:** *YOUTUBE, OPENLIBRARY, INTERNETARCHIVE, SOULSEEK, LIBGEN, ALLDEBRID, TELEGRAM, BANDCAMP*
    • +- **Plugin integration:** *YOUTUBE, OPENLIBRARY, INTERNETARCHIVE, SOULSEEK, LIBGEN, ALLDEBRID, TELEGRAM, BANDCAMP*
  • Module Mixing: *[Playwright](https://github.com/microsoft/playwright), [yt-dlp](https://github.com/yt-dlp/yt-dlp), [typer](https://github.com/fastapi/typer)*
    • -
  • Optional stacks: Telethon (Telegram), aioslsk (Soulseek), and the FlorenceVision tooling install automatically when you configure the corresponding provider/tool blocks. +
  • Optional stacks: Telethon (Telegram), aioslsk (Soulseek), and the FlorenceVision tooling install automatically when you configure the corresponding plugin/tool blocks.
  • MPV Manager: Play audio, video, and even images in a custom designed MPV with trimming, screenshotting, and more built right in!
  • Supports remote access and networked setups for offsite servers and sharing workflows.