From 323c24f4f4059c3d13ef047250c03a7f01ae8534 Mon Sep 17 00:00:00 2001 From: Nose Date: Tue, 28 Apr 2026 22:20:54 -0700 Subject: [PATCH] updating and refining plugin system refactor --- API/HydrusNetwork.py | 35 +- CLI.py | 141 ++- MPV/LUA/main.lua | 6 + Provider/example_provider.py | 261 +--- Provider/metadata_provider.py | 2030 +----------------------------- Provider/tidal_manifest.py | 345 +----- ProviderCore/base.py | 265 +++- ProviderCore/registry.py | 191 ++- SYS/detail_view_helpers.py | 86 +- SYS/pipeline.py | 44 +- SYS/plugin_config.py | 12 - SYS/result_table.py | 71 +- SYS/rich_display.py | 24 +- Store/HydrusNetwork.py | 35 +- TUI/modalscreen/config_modal.py | 56 +- TUI/modalscreen/search.py | 6 +- cmdlet/_shared.py | 12 +- cmdlet/add_file.py | 51 +- cmdlet/download_file.py | 26 +- cmdlet/get_tag.py | 44 +- cmdlet/search_file.py | 54 +- cmdnat/matrix.py | 2 +- cmdnat/pipe.py | 306 +++-- docs/ftp_plugin_tutorial.md | 56 +- docs/scp_plugin_tutorial.md | 46 +- plugins/README.md | 7 +- plugins/alldebrid/__init__.py | 80 +- plugins/example_provider.py | 260 +++- plugins/ftp/__init__.py | 335 +++-- plugins/libgen/__init__.py | 10 +- plugins/metadata_provider.py | 2034 ++++++++++++++++++++++++++++++- plugins/scp/__init__.py | 324 +++-- plugins/tidal_manifest.py | 344 +++++- 33 files changed, 4287 insertions(+), 3312 deletions(-) diff --git a/API/HydrusNetwork.py b/API/HydrusNetwork.py index 20bd566..b948631 100644 --- a/API/HydrusNetwork.py +++ b/API/HydrusNetwork.py @@ -1,37 +1,6 @@ -"""Hydrus API helpers and export utilities.""" +"""Compatibility shim for the Hydrus plugin-owned API module.""" -from __future__ import annotations - -import base64 -import http.client -import json -import os -import re -import shutil -import subprocess -import sys -import time -from collections import deque - -from SYS.logger import log -from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS as GLOBAL_SUPPORTED_EXTENSIONS -import tempfile -import logging -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast -from urllib.parse import urlsplit, urlencode, quote, urlunsplit, unquote -import httpx - -logger = logging.getLogger(__name__) - -from SYS.utils import ( - decode_cbor, - jsonify, - ensure_directory, - unique_path, -) -from .HTTP import HTTPClient +from plugins.hydrusnetwork.api import * # noqa: F401,F403 class HydrusRequestError(RuntimeError): diff --git a/CLI.py b/CLI.py index 114f782..b86695b 100644 --- a/CLI.py +++ b/CLI.py @@ -506,32 +506,28 @@ class CmdletIntrospection: if normalized_arg == "plugin": canonical_cmd = (cmd_name or "").replace("_", "-").lower() try: - from ProviderCore.registry import ( - list_search_plugin_names, - list_upload_plugin_names, - ) + from ProviderCore.registry import list_plugin_names_with_capability except Exception: - list_search_plugin_names = None # type: ignore - list_upload_plugin_names = None # type: ignore + list_plugin_names_with_capability = None # type: ignore - provider_choices: List[str] = [] + plugin_choices: List[str] = [] - if canonical_cmd in {"add-file"} and list_upload_plugin_names is not None: - return list_upload_plugin_names() or [] + if canonical_cmd in {"add-file"} and list_plugin_names_with_capability is not None: + return list_plugin_names_with_capability("upload") or [] - if list_search_plugin_names is not None: - provider_choices = list_search_plugin_names() or [] + if list_plugin_names_with_capability is not None: + plugin_choices = list_plugin_names_with_capability("search") or [] - if provider_choices: - return provider_choices + if plugin_choices: + return plugin_choices if normalized_arg == "scrape": try: - from plugins.metadata_provider import list_metadata_providers + from plugins.metadata_provider import list_metadata_plugins - meta_providers = list_metadata_providers(config) or {} - if meta_providers: - return sorted(meta_providers.keys()) + metadata_plugins = list_metadata_plugins(config) or {} + if metadata_plugins: + return sorted(metadata_plugins.keys()) except Exception: pass @@ -704,6 +700,77 @@ class CmdletCompleter(Completer): return tokens[idx + 1] return None + @staticmethod + def _selected_plugin_name(cmd_name: str, stage_tokens: Sequence[str]) -> Optional[str]: + canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower() + if canonical_cmd not in {"search-file", "add-file", "download-file"}: + return None + return CmdletCompleter._flag_value(stage_tokens, "-plugin", "--plugin") + + @staticmethod + def _plugin_instance_choices(plugin_name: Optional[str], config: Dict[str, Any]) -> List[str]: + plugin_key = str(plugin_name or "").strip().lower() + if not plugin_key: + return [] + + try: + from ProviderCore.registry import get_plugin_class + except Exception: + return [] + + plugin_class = get_plugin_class(plugin_key) + if plugin_class is None: + return [] + + try: + plugin = plugin_class(config) + except Exception: + return [] + + try: + instances = plugin.configured_instances() + except Exception: + return [] + + out: List[str] = [] + seen: Set[str] = set() + for value in instances or []: + text = str(value or "").strip() + lowered = text.lower() + if not text or lowered in seen: + continue + seen.add(lowered) + out.append(text) + return out + + def _filter_stage_arg_names( + self, + *, + cmd_name: str, + stage_tokens: Sequence[str], + config: Dict[str, Any], + arg_names: List[str], + ) -> List[str]: + if not arg_names: + return [] + + canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower() + plugin_name = self._selected_plugin_name(canonical_cmd, stage_tokens) + instance_choices = self._plugin_instance_choices(plugin_name, config) + has_named_instances = bool(instance_choices) + + filtered: List[str] = [] + for arg in arg_names: + logical = str(arg or "").lstrip("-").strip().lower() + if logical == "instance": + if not plugin_name or not has_named_instances: + continue + if canonical_cmd == "search-file" and logical == "open": + if str(plugin_name or "").strip().lower() != "alldebrid": + continue + filtered.append(arg) + return filtered + def get_completions( self, document: Document, @@ -742,7 +809,12 @@ class CmdletCompleter(Completer): if cmd_name not in self.cmdlet_names: return - arg_names = self._cmdlet_args(cmd_name, config) + arg_names = self._filter_stage_arg_names( + cmd_name=cmd_name, + stage_tokens=stage_tokens, + config=config, + arg_names=self._cmdlet_args(cmd_name, config), + ) seen_logicals: Set[str] = set() for arg in arg_names: arg_low = arg.lower() @@ -779,6 +851,8 @@ class CmdletCompleter(Completer): if cmd_name == "search-file": provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin") + selected_plugin = self._selected_plugin_name(cmd_name, stage_tokens) + query_specs = self._query_args(cmd_name, config) query_flag_index = -1 for idx, tok in enumerate(stage_tokens): @@ -886,28 +960,43 @@ class CmdletCompleter(Completer): yield Completion(suggestion, start_position=start_pos) return - choices = self._arg_choices( - cmd_name=cmd_name, - arg_name=prev_token, - config=config, - force=False, - ) + normalized_prev = prev_token.lstrip("-").strip().lower() + choices: List[str] = [] + if normalized_prev == "instance" and selected_plugin: + choices = self._plugin_instance_choices(selected_plugin, config) + if not choices: + choices = self._arg_choices( + cmd_name=cmd_name, + arg_name=prev_token, + config=config, + force=False, + ) if choices: choice_list = choices - normalized_prev = prev_token.lstrip("-").strip().lower() if normalized_prev in {"plugin", "provider"} and current_token: current_lower = current_token.lower() filtered = [c for c in choices if current_lower in c.lower()] if filtered: choice_list = filtered + if normalized_prev == "instance" and current_token: + current_lower = current_token.lower() + filtered = [c for c in choice_list if current_lower in c.lower()] + if filtered: + choice_list = filtered + for choice in choice_list: yield Completion(choice, start_position=-len(current_token)) # Example: if the user has typed `download-file -url ...`, then `url` # is considered used and should not be suggested again (even as `--url`). return - arg_names = self._cmdlet_args(cmd_name, config) + arg_names = self._filter_stage_arg_names( + cmd_name=cmd_name, + stage_tokens=stage_tokens, + config=config, + arg_names=self._cmdlet_args(cmd_name, config), + ) used_logicals = self._used_arg_logicals(cmd_name, stage_tokens, config) logical_seen: Set[str] = set() for arg in arg_names: diff --git a/MPV/LUA/main.lua b/MPV/LUA/main.lua index b31d51b..ce0f777 100644 --- a/MPV/LUA/main.lua +++ b/MPV/LUA/main.lua @@ -4105,6 +4105,9 @@ function M._apply_web_subtitle_load_defaults(reason) if target == '' or not _is_http_url(target) then return false end + if not _is_ytdlp_url(target) then + return false + end M._prepare_ytdl_format_for_web_load(target, reason or 'on-load') @@ -4250,6 +4253,9 @@ function M._ensure_current_subtitles_visible(reason) if (not current or current == '') and (path == '' or not _is_http_url(path)) then return false end + if (not current or current == '') and (path == '' or not _is_ytdlp_url(path)) then + return false + end local track_id, source, already_selected = M._find_subtitle_track_candidate() if not track_id then diff --git a/Provider/example_provider.py b/Provider/example_provider.py index d3bc824..46d5543 100644 --- a/Provider/example_provider.py +++ b/Provider/example_provider.py @@ -1,259 +1,8 @@ -"""Example plugin that uses the new `ResultTable` API. +"""Legacy compatibility shim for the strict adapter example module. -This module demonstrates a minimal provider adapter that yields `ResultModel` -instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer -(`render_table`) for demonstration. - -Run this to see sample output: - python -m Provider.example_provider - -Example usage (piped selector): - plugin-table -plugin example -sample | select -select 1 | add-file -store default +The active implementation now lives in ``plugins.example_provider`` so the +plugin namespace owns the example adapter module. Keep this file only to avoid +breaking old imports while the legacy ``Provider`` package is phased out. """ -from __future__ import annotations -from pathlib import Path -from typing import Any, Dict, Iterable, List - -from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column - - -SAMPLE_ITEMS = [ - { - "name": "Book of Awe.pdf", - "path": "sample/Book of Awe.pdf", - "ext": "pdf", - "size": 1024000, - "source": "example", - }, - { - "name": "Song of Joy.mp3", - "path": "sample/Song of Joy.mp3", - "ext": "mp3", - "size": 5120000, - "source": "example", - }, - { - "name": "Cover Image.jpg", - "path": "sample/Cover Image.jpg", - "ext": "jpg", - "size": 20480, - "source": "example", - }, -] - - -def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]: - """Convert provider-specific items into `ResultModel` instances. - - This adapter enforces the strict API requirement: it yields only - `ResultModel` instances (no legacy dict objects). - """ - for it in items: - title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "") - yield ResultModel( - title=str(title), - path=str(it.get("path")) if it.get("path") else None, - ext=str(it.get("ext")) if it.get("ext") else None, - size_bytes=int(it.get("size")) if it.get("size") is not None else None, - metadata=dict(it), - source=str(it.get("source")) if it.get("source") else "example", - ) - - -# Columns are intentionally *not* mandated. Create a factory that inspects -# sample rows and builds only columns that make sense for the provider data. -from SYS.result_table_api import metadata_column - - -def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: - cols: List[ColumnSpec] = [title_column()] - - # If any row has an extension, include Ext column - if any(getattr(r, "ext", None) for r in rows): - cols.append(ext_column()) - - # If any row has size, include Size column - if any(getattr(r, "size_bytes", None) for r in rows): - cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v))) - - # Add any top-level metadata keys discovered (up to 3) as optional columns - seen_keys = [] - for r in rows: - for k in (r.metadata or {}).keys(): - if k in ("name", "title", "path"): - continue - if k not in seen_keys: - seen_keys.append(k) - if len(seen_keys) >= 3: - break - if len(seen_keys) >= 3: - break - - for k in seen_keys: - cols.append(metadata_column(k)) - - return cols - - -# Selection function: cmdlets rely on this to build selector args when the user -# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer -# -path if available, otherwise fall back to -title. -def selection_fn(row: ResultModel) -> List[str]: - if row.path: - return ["-path", row.path] - return ["-title", row.title] - - -# Register the plugin with the registry so callers can discover it by name -from SYS.result_table_adapters import register_plugin -register_plugin( - "example", - adapter, - columns=columns_factory, - selection_fn=selection_fn, - metadata={"description": "Example provider demonstrating dynamic columns and selectors"}, -) - - -def _format_size(size: Any) -> str: - try: - s = int(size) - except Exception: - return "" - if s >= 1024 ** 3: - return f"{s / (1024 ** 3):.2f} GB" - if s >= 1024 ** 2: - return f"{s / (1024 ** 2):.2f} MB" - if s >= 1024: - return f"{s / 1024:.2f} KB" - return f"{s} B" - - -def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str: - """Render a simple ASCII table of `rows` using `columns`. - - This is intentionally very small and dependency-free for demonstration. - Renderers in the project should implement the `Renderer` protocol. - """ - rows = list(rows) - - # Build cell matrix (strings) - matrix: List[List[str]] = [] - for r in rows: - cells: List[str] = [] - for col in columns: - raw = col.extractor(r) - if col.format_fn: - try: - cell = col.format_fn(raw) - except Exception: - cell = str(raw or "") - else: - cell = str(raw or "") - cells.append(cell) - matrix.append(cells) - - # Compute column widths as max(header, content) - headers = [c.header for c in columns] - widths = [len(h) for h in headers] - for row_cells in matrix: - for i, cell in enumerate(row_cells): - widths[i] = max(widths[i], len(cell)) - - # Helper to format a row - def fmt_row(cells: List[str]) -> str: - return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) - - lines: List[str] = [] - lines.append(fmt_row(headers)) - lines.append("-+-".join("-" * w for w in widths)) - for row_cells in matrix: - lines.append(fmt_row(row_cells)) - - return "\n".join(lines) - - -# Rich-based renderer (returns a Rich Table renderable) -def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]): - """Render rows as a `rich.table.Table` for terminal output. - - Returns the Table object; callers may `Console.print(table)` to render. - """ - try: - from rich.table import Table as RichTable - except Exception as exc: # pragma: no cover - rare if rich missing - raise RuntimeError("rich is required for rich renderer") from exc - - table = RichTable(show_header=True, header_style="bold") - for col in columns: - table.add_column(col.header) - - for r in rows: - cells: List[str] = [] - for col in columns: - raw = col.extractor(r) - if col.format_fn: - try: - cell = col.format_fn(raw) - except Exception: - cell = str(raw or "") - else: - cell = str(raw or "") - cells.append(cell) - table.add_row(*cells) - - return table - - -def demo() -> None: - rows = list(adapter(SAMPLE_ITEMS)) - table = render_table_rich(rows, columns_factory(rows)) - try: - from rich.console import Console - except Exception: - # Fall back to plain printing if rich is not available - print("Example provider output:") - print(render_table(rows, columns_factory(rows))) - return - - console = Console() - console.print("Example provider output:") - console.print(table) - - -def demo_with_selection(idx: int = 0) -> None: - """Demonstrate how a cmdlet would use plugin registration and selection args. - - - Fetch the registered plugin by name - - Build rows via adapter - - Render the table - - Show the selection args for the chosen row; these are the args a cmdlet - would append when the user picks that row. - """ - from SYS.result_table_adapters import get_plugin - - provider = get_plugin("example") - rows = list(provider.adapter(SAMPLE_ITEMS)) - cols = provider.get_columns(rows) - - # Render - try: - from rich.console import Console - except Exception: - print(render_table(rows, cols)) - sel_args = provider.selection_args(rows[idx]) - print("Selection args for row", idx, "->", sel_args) - return - - console = Console() - console.print("Example provider output:") - console.print(render_table_rich(rows, cols)) - - # Selection args example - sel = provider.selection_args(rows[idx]) - console.print("Selection args for row", idx, "->", sel) - - -if __name__ == "__main__": - demo() +from plugins.example_provider import * # noqa: F401,F403 diff --git a/Provider/metadata_provider.py b/Provider/metadata_provider.py index 0c06780..a9278b6 100644 --- a/Provider/metadata_provider.py +++ b/Provider/metadata_provider.py @@ -1,2026 +1,8 @@ -from __future__ import annotations +"""Legacy compatibility shim for metadata helpers. -from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Type, cast -import html as html_std -import re -import sys -import json -import subprocess +The active implementation now lives in ``plugins.metadata_provider`` so the +plugin namespace owns runtime metadata scraping. Keep this file only to avoid +breaking old imports while the legacy ``Provider`` package is phased out. +""" -from API.HTTP import HTTPClient -from API.requests_client import get_requests_session -from ProviderCore.base import SearchResult -try: - from plugins.tidal import Tidal -except ImportError: # pragma: no cover - optional - Tidal = None -from API.Tidal import ( - build_track_tags, - extract_artists, - stringify, -) -try: # Optional dependency for IMDb scraping - from imdbinfo.services import search_title # type: ignore -except ImportError: # pragma: no cover - optional - search_title = None # type: ignore[assignment] - -from SYS.logger import log, debug -from SYS.metadata import imdb_tag -from SYS.json_table import normalize_record - -try: # Optional dependency - import musicbrainzngs # type: ignore -except ImportError: # pragma: no cover - optional - musicbrainzngs = None - -try: # Optional dependency - import yt_dlp # type: ignore -except ImportError: # pragma: no cover - optional - yt_dlp = None - - -def _dedup_text_values(values: List[str]) -> List[str]: - out: List[str] = [] - seen: set[str] = set() - for value in values or []: - if value is None: - continue - text = str(value).strip() - if not text: - continue - key = text.lower() - if key in seen: - continue - seen.add(key) - out.append(text) - return out - - -def _filter_default_scraped_tags(tags: List[str]) -> List[str]: - blocked = {"title", "artist", "source"} - out: List[str] = [] - seen: set[str] = set() - for tag in tags or []: - text = str(tag or "").strip() - if not text: - continue - namespace = text.split(":", 1)[0].strip().lower() if ":" in text else "" - if namespace in blocked: - continue - key = text.lower() - if key in seen: - continue - seen.add(key) - out.append(text) - return out - - -class MetadataProvider(ABC): - """Base class for metadata providers (music, movies, books, etc.).""" - - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: - self.config = config or {} - - @property - def name(self) -> str: - return self.__class__.__name__.replace("Provider", "").lower() - - @abstractmethod - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - """Return a list of candidate metadata records.""" - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - """Convert a result item into a list of tags.""" - tags: List[str] = [] - title = item.get("title") - artist = item.get("artist") - album = item.get("album") - year = item.get("year") - - if title: - tags.append(f"title:{title}") - if artist: - tags.append(f"artist:{artist}") - if album: - tags.append(f"album:{album}") - if year: - tags.append(f"year:{year}") - - tags.append(f"source:{self.name}") - return tags - - def search_tags(self, query: str, limit: int = 1) -> List[str]: - """Return tags for the best match from `search(query)`. - - Providers can override this when tags should be extracted differently from - the default search->first-item->to_tags flow. - """ - - try: - items = self.search(query, limit=max(1, int(limit))) - except Exception: - return [] - if not items: - return [] - try: - return [str(t) for t in self.to_tags(items[0]) if t is not None] - except Exception: - return [] - - def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: - """Return provider-specific identifier query text from parsed identifiers.""" - - _ = identifiers - return None - - def combined_query( - self, - *, - title_hint: Optional[str], - artist_hint: Optional[str], - ) -> Optional[str]: - """Return provider-specific title+artist query text.""" - - _ = title_hint - _ = artist_hint - return None - - def extract_url_query(self, result: Any, get_field: Any) -> Optional[str]: - """Return provider-specific URL query derived from a piped result.""" - - _ = result - _ = get_field - return None - - def emits_direct_tags(self) -> bool: - """True when provider should skip selection table and emit tags directly.""" - - return False - - def default_subject_scrape_priority(self) -> int: - """Priority used when `get-tag -scrape` is invoked without an explicit provider.""" - - return 0 - - def url_scrape_priority(self, url: str) -> int: - """Priority for handling a raw URL passed to `get-tag -scrape `.""" - - _ = url - return 0 - - def resolve_subject_query( - self, - result: Any, - get_field: Any, - *, - backend: Any = None, - file_hash: Optional[str] = None, - ) -> Optional[str]: - """Resolve a provider-specific query from the current subject/result.""" - - _ = backend - _ = file_hash - return self.extract_url_query(result, get_field) - - def prefers_store_tag_overwrite(self) -> bool: - """Whether direct subject scrapes should replace the store tag set.""" - - return False - - def filter_tags_for_selection(self, tags: List[str]) -> List[str]: - """Filter scraped tags before presenting a selectable metadata row.""" - - return _filter_default_scraped_tags(tags) - - def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: - """Filter scraped tags before applying them to an existing store-backed item.""" - - return self.filter_tags_for_selection(tags) - - def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: - """Return a URL scrape payload for `get-tag -scrape ` when supported.""" - - items = self.search(url, limit=1) - if not items: - return None - item = items[0] if isinstance(items[0], dict) else {} - try: - tags = [str(t) for t in self.to_tags(item) if t is not None] - except Exception: - tags = [] - return { - "title": item.get("title"), - "tag": _dedup_text_values(tags), - "formats": [], - "playlist_items": [], - } - - -class ITunesProvider(MetadataProvider): - """Metadata provider using the iTunes Search API.""" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - params = { - "term": query, - "media": "music", - "entity": "song", - "limit": limit - } - try: - resp = get_requests_session().get( - "https://itunes.apple.com/search", - params=params, - timeout=10 - ) - resp.raise_for_status() - results = resp.json().get("results", []) - except Exception as exc: - log(f"iTunes search failed: {exc}", file=sys.stderr) - return [] - - items: List[Dict[str, Any]] = [] - for r in results: - item = { - "title": r.get("trackName"), - "artist": r.get("artistName"), - "album": r.get("collectionName"), - "year": str(r.get("releaseDate", - ""))[:4], - "provider": self.name, - "raw": r, - } - items.append(item) - debug(f"iTunes returned {len(items)} items for '{query}'") - return items - - def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: - return identifiers.get("musicbrainz") or identifiers.get("musicbrainzalbum") - - def combined_query( - self, - *, - title_hint: Optional[str], - artist_hint: Optional[str], - ) -> Optional[str]: - title_text = str(title_hint or "").strip() - artist_text = str(artist_hint or "").strip() - if not title_text or not artist_text: - return None - return f"{title_text} {artist_text}" - - -class OpenLibraryMetadataProvider(MetadataProvider): - """Metadata provider for OpenLibrary book metadata.""" - - @property - def name(self) -> str: # type: ignore[override] - return "openlibrary" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - query_clean = (query or "").strip() - if not query_clean: - return [] - - try: - # Prefer ISBN-specific search when the query looks like one - if query_clean.replace("-", - "").isdigit() and len(query_clean.replace("-", - "")) in ( - 10, - 13, - ): - q = f"isbn:{query_clean.replace('-', '')}" - else: - q = query_clean - - resp = get_requests_session().get( - "https://openlibrary.org/search.json", - params={ - "q": q, - "limit": limit - }, - timeout=10, - ) - resp.raise_for_status() - data = resp.json() - except Exception as exc: - log(f"OpenLibrary search failed: {exc}", file=sys.stderr) - return [] - - items: List[Dict[str, Any]] = [] - for doc in data.get("docs", [])[:limit]: - authors = doc.get("author_name") or [] - publisher = "" - publishers = doc.get("publisher") or [] - if isinstance(publishers, list) and publishers: - publisher = publishers[0] - - # Prefer 13-digit ISBN when available, otherwise 10-digit - isbn_list = doc.get("isbn") or [] - isbn_13 = next((i for i in isbn_list if len(str(i)) == 13), None) - isbn_10 = next((i for i in isbn_list if len(str(i)) == 10), None) - - # Derive OLID from key - olid = "" - key = doc.get("key", "") - if isinstance(key, str) and key: - olid = key.split("/")[-1] - - items.append( - { - "title": doc.get("title") or "", - "artist": ", ".join(authors) if authors else "", - "album": publisher, - "year": str(doc.get("first_publish_year") or ""), - "provider": self.name, - "authors": authors, - "publisher": publisher, - "identifiers": { - "isbn_13": isbn_13, - "isbn_10": isbn_10, - "openlibrary": olid, - "oclc": (doc.get("oclc_numbers") or [None])[0], - "lccn": (doc.get("lccn") or [None])[0], - }, - "description": None, - } - ) - - return items - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - tags: List[str] = [] - title = item.get("title") - authors = item.get("authors") or [] - publisher = item.get("publisher") - year = item.get("year") - description = item.get("description") or "" - - if title: - tags.append(f"title:{title}") - for author in authors: - if author: - tags.append(f"author:{author}") - if publisher: - tags.append(f"publisher:{publisher}") - if year: - tags.append(f"year:{year}") - if description: - tags.append(f"description:{description[:200]}") - - identifiers = item.get("identifiers") or {} - for key, value in identifiers.items(): - if value: - tags.append(f"{key}:{value}") - - tags.append(f"source:{self.name}") - return tags - - def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: - return ( - identifiers.get("isbn_13") - or identifiers.get("isbn_10") - or identifiers.get("isbn") - or identifiers.get("openlibrary") - ) - - -class GoogleBooksMetadataProvider(MetadataProvider): - """Metadata provider for Google Books volumes API.""" - - @property - def name(self) -> str: # type: ignore[override] - return "googlebooks" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - query_clean = (query or "").strip() - if not query_clean: - return [] - - # Prefer ISBN queries when possible - if query_clean.replace("-", - "").isdigit() and len(query_clean.replace("-", - "")) in (10, - 13): - q = f"isbn:{query_clean.replace('-', '')}" - else: - q = query_clean - - try: - resp = get_requests_session().get( - "https://www.googleapis.com/books/v1/volumes", - params={ - "q": q, - "maxResults": limit - }, - timeout=10, - ) - resp.raise_for_status() - payload = resp.json() - except Exception as exc: - log(f"Google Books search failed: {exc}", file=sys.stderr) - return [] - - items: List[Dict[str, Any]] = [] - for volume in payload.get("items", [])[:limit]: - info = volume.get("volumeInfo") or {} - authors = info.get("authors") or [] - publisher = info.get("publisher", "") - published_date = info.get("publishedDate", "") - year = str(published_date)[:4] if published_date else "" - - identifiers_raw = info.get("industryIdentifiers") or [] - identifiers: Dict[str, - Optional[str]] = { - "googlebooks": volume.get("id") - } - for ident in identifiers_raw: - if not isinstance(ident, dict): - continue - ident_type = ident.get("type", "").lower() - ident_value = ident.get("identifier") - if not ident_value: - continue - if ident_type == "isbn_13": - identifiers.setdefault("isbn_13", ident_value) - elif ident_type == "isbn_10": - identifiers.setdefault("isbn_10", ident_value) - else: - identifiers.setdefault(ident_type, ident_value) - - items.append( - { - "title": info.get("title") or "", - "artist": ", ".join(authors) if authors else "", - "album": publisher, - "year": year, - "provider": self.name, - "authors": authors, - "publisher": publisher, - "identifiers": identifiers, - "description": info.get("description", - ""), - } - ) - - return items - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - tags: List[str] = [] - title = item.get("title") - authors = item.get("authors") or [] - publisher = item.get("publisher") - year = item.get("year") - description = item.get("description") or "" - - if title: - tags.append(f"title:{title}") - for author in authors: - if author: - tags.append(f"author:{author}") - if publisher: - tags.append(f"publisher:{publisher}") - if year: - tags.append(f"year:{year}") - if description: - tags.append(f"description:{description[:200]}") - - identifiers = item.get("identifiers") or {} - for key, value in identifiers.items(): - if value: - tags.append(f"{key}:{value}") - - tags.append(f"source:{self.name}") - return tags - - def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: - return ( - identifiers.get("isbn_13") - or identifiers.get("isbn_10") - or identifiers.get("isbn") - or identifiers.get("openlibrary") - ) - - -class ISBNsearchMetadataProvider(MetadataProvider): - """Metadata provider that scrapes isbnsearch.org by ISBN. - - This is a best-effort HTML scrape. It expects the query to be an ISBN. - """ - - @property - def name(self) -> str: # type: ignore[override] - return "isbnsearch" - - @staticmethod - def _strip_html_to_text(raw: str) -> str: - s = html_std.unescape(str(raw or "")) - s = re.sub(r"(?i)", "\n", s) - s = re.sub(r"<[^>]+>", " ", s) - s = re.sub(r"\s+", " ", s) - return s.strip() - - @staticmethod - def _clean_isbn(query: str) -> str: - s = str(query or "").strip() - if not s: - return "" - s = s.replace("isbn:", "").replace("ISBN:", "") - s = re.sub(r"[^0-9Xx]", "", s).upper() - if len(s) in (10, 13): - return s - # Try to locate an ISBN-like token inside the query. - m = re.search(r"\b(?:97[89])?\d{9}[\dXx]\b", s) - return str(m.group(0)).upper() if m else "" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - _ = limit - isbn = self._clean_isbn(query) - if not isbn: - return [] - - url = f"https://isbnsearch.org/isbn/{isbn}" - try: - resp = get_requests_session().get(url, timeout=10) - resp.raise_for_status() - html = str(resp.text or "") - if not html: - return [] - except Exception as exc: - log(f"ISBNsearch scrape failed: {exc}", file=sys.stderr) - return [] - - title = "" - m_title = re.search(r"(?is)]*>(.*?)", html) - if m_title: - title = self._strip_html_to_text(m_title.group(1)) - - raw_fields: Dict[str, - str] = {} - strong_matches = list(re.finditer(r"(?is)]*>(.*?)", html)) - for idx, m in enumerate(strong_matches): - label_raw = self._strip_html_to_text(m.group(1)) - label = str(label_raw or "").strip() - if not label: - continue - if label.endswith(":"): - label = label[:-1].strip() - - chunk_start = m.end() - # Stop at next or end of document. - chunk_end = ( - strong_matches[idx + 1].start() if - (idx + 1) < len(strong_matches) else len(html) - ) - chunk = html[chunk_start:chunk_end] - # Prefer stopping within the same paragraph when possible. - m_end = re.search(r"(?is)(

|)", chunk) - if m_end: - chunk = chunk[:m_end.start()] - - val_text = self._strip_html_to_text(chunk) - if not val_text: - continue - raw_fields[label] = val_text - - def _get(*labels: str) -> str: - for lab in labels: - for k, v in raw_fields.items(): - if str(k).strip().lower() == str(lab).strip().lower(): - return str(v or "").strip() - return "" - - # Map common ISBNsearch labels. - author_text = _get("Author", "Authors", "Author(s)") - publisher = _get("Publisher") - published = _get("Published", "Publication Date", "Publish Date") - language = _get("Language") - pages = _get("Pages") - isbn_13 = _get("ISBN-13", "ISBN13") - isbn_10 = _get("ISBN-10", "ISBN10") - - year = "" - if published: - m_year = re.search(r"\b(\d{4})\b", published) - year = str(m_year.group(1)) if m_year else "" - - authors: List[str] = [] - if author_text: - # Split on common separators; keep multi-part names intact. - for part in re.split(r"\s*(?:,|;|\band\b|\&|\|)\s*", - author_text, - flags=re.IGNORECASE): - p = str(part or "").strip() - if p: - authors.append(p) - - # Prefer parsed title, but fall back to og:title if needed. - if not title: - m_og = re.search( - r"(?is)]*property=['\"]og:title['\"][^>]*content=['\"](.*?)['\"][^>]*>", - html, - ) - if m_og: - title = self._strip_html_to_text(m_og.group(1)) - - # Ensure ISBN tokens are normalized. - isbn_tokens: List[str] = [] - for token in [isbn_13, isbn_10, isbn]: - t = self._clean_isbn(token) - if t and t not in isbn_tokens: - isbn_tokens.append(t) - - item: Dict[str, - Any] = { - "title": title or "", - # Keep UI columns compatible with the generic metadata table. - "artist": ", ".join(authors) if authors else "", - "album": publisher or "", - "year": year or "", - "provider": self.name, - "authors": authors, - "publisher": publisher or "", - "language": language or "", - "pages": pages or "", - "identifiers": { - "isbn_13": - next((t for t in isbn_tokens if len(t) == 13), - None), - "isbn_10": - next((t for t in isbn_tokens if len(t) == 10), - None), - }, - "raw_fields": raw_fields, - } - - # Only return usable items. - if not item.get("title") and not any(item["identifiers"].values()): - return [] - - return [item] - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - tags: List[str] = [] - - title = str(item.get("title") or "").strip() - if title: - tags.append(f"title:{title}") - - authors = item.get("authors") or [] - if isinstance(authors, list): - for a in authors: - a = str(a or "").strip() - if a: - tags.append(f"author:{a}") - - publisher = str(item.get("publisher") or "").strip() - if publisher: - tags.append(f"publisher:{publisher}") - - year = str(item.get("year") or "").strip() - if year: - tags.append(f"year:{year}") - - language = str(item.get("language") or "").strip() - if language: - tags.append(f"language:{language}") - - identifiers = item.get("identifiers") or {} - if isinstance(identifiers, dict): - for key in ("isbn_13", "isbn_10"): - val = identifiers.get(key) - if val: - tags.append(f"isbn:{val}") - - tags.append(f"source:{self.name}") - - # Dedup case-insensitively, preserve order. - seen: set[str] = set() - out: List[str] = [] - for t in tags: - s = str(t or "").strip() - if not s: - continue - k = s.lower() - if k in seen: - continue - seen.add(k) - out.append(s) - return out - - -class MusicBrainzMetadataProvider(MetadataProvider): - """Metadata provider for MusicBrainz recordings.""" - - @property - def name(self) -> str: # type: ignore[override] - return "musicbrainz" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - if not musicbrainzngs: - log( - "musicbrainzngs is not installed; skipping MusicBrainz scrape", - file=sys.stderr - ) - return [] - - q = (query or "").strip() - if not q: - return [] - - try: - # Ensure user agent is set (required by MusicBrainz) - musicbrainzngs.set_useragent("Medeia-Macina", "0.1") - except Exception: - pass - - try: - resp = musicbrainzngs.search_recordings(query=q, limit=limit) - recordings = resp.get("recording-list") or resp.get("recordings") or [] - except Exception as exc: - log(f"MusicBrainz search failed: {exc}", file=sys.stderr) - return [] - - items: List[Dict[str, Any]] = [] - for rec in recordings[:limit]: - if not isinstance(rec, dict): - continue - title = rec.get("title") or "" - - artist = "" - artist_credit = rec.get("artist-credit") or rec.get("artist_credit") - if isinstance(artist_credit, list) and artist_credit: - first = artist_credit[0] - if isinstance(first, dict): - artist = first.get("name") or first.get("artist", - {}).get("name", - "") - elif isinstance(first, str): - artist = first - - album = "" - release_list = rec.get("release-list") or rec.get("releases" - ) or rec.get("release") - if isinstance(release_list, list) and release_list: - first_rel = release_list[0] - if isinstance(first_rel, dict): - album = first_rel.get("title", "") or "" - release_date = first_rel.get("date") or "" - else: - album = str(first_rel) - release_date = "" - else: - release_date = rec.get("first-release-date") or "" - - year = str(release_date)[:4] if release_date else "" - mbid = rec.get("id") or "" - - items.append( - { - "title": title, - "artist": artist, - "album": album, - "year": year, - "provider": self.name, - "mbid": mbid, - "raw": rec, - } - ) - - return items - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - tags = super().to_tags(item) - mbid = item.get("mbid") - if mbid: - tags.append(f"musicbrainz:{mbid}") - return tags - - def combined_query( - self, - *, - title_hint: Optional[str], - artist_hint: Optional[str], - ) -> Optional[str]: - title_text = str(title_hint or "").strip() - artist_text = str(artist_hint or "").strip() - if not title_text or not artist_text: - return None - return f'recording:"{title_text}" AND artist:"{artist_text}"' - - -class ImdbMetadataProvider(MetadataProvider): - """Metadata provider for IMDb titles (movies/series/episodes).""" - - @property - def name(self) -> str: # type: ignore[override] - return "imdb" - - @staticmethod - def _extract_imdb_id(text: str) -> str: - raw = str(text or "").strip() - if not raw: - return "" - - # Exact tt123 pattern - m = re.search(r"(tt\d+)", raw, re.IGNORECASE) - if m: - imdb_id = m.group(1).lower() - return imdb_id if imdb_id.startswith("tt") else f"tt{imdb_id}" - - # Bare numeric IDs (e.g., "0118883") - if raw.isdigit() and len(raw) >= 6: - return f"tt{raw}" - - # Last-resort: extract first digit run - m_digits = re.search(r"(\d{6,})", raw) - if m_digits: - return f"tt{m_digits.group(1)}" - - return "" - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - q = (query or "").strip() - if not q: - return [] - - imdb_id = self._extract_imdb_id(q) - if imdb_id: - try: - data = imdb_tag(imdb_id) - raw_tags = data.get("tag") if isinstance(data, dict) else [] - title = None - year = None - if isinstance(raw_tags, list): - for tag in raw_tags: - if not isinstance(tag, str): - continue - if tag.startswith("title:"): - title = tag.split(":", 1)[1] - elif tag.startswith("year:"): - year = tag.split(":", 1)[1] - return [ - { - "title": title or imdb_id, - "artist": "", - "album": "", - "year": str(year or ""), - "provider": self.name, - "imdb_id": imdb_id, - "raw": data, - } - ] - except Exception as exc: - log(f"IMDb lookup failed: {exc}", file=sys.stderr) - return [] - - if search_title is None: - log("imdbinfo is not installed; skipping IMDb scrape", file=sys.stderr) - return [] - - try: - search_result = search_title(q) - titles = getattr(search_result, "titles", None) or [] - except Exception as exc: - log(f"IMDb search failed: {exc}", file=sys.stderr) - return [] - - items: List[Dict[str, Any]] = [] - for entry in titles[:limit]: - imdb_id = self._extract_imdb_id( - getattr(entry, "imdb_id", None) - or getattr(entry, "imdbId", None) - or getattr(entry, "id", None) - ) - title = getattr(entry, "title", "") or getattr(entry, "title_localized", "") - year = str(getattr(entry, "year", "") or "")[:4] - kind = getattr(entry, "kind", "") or "" - rating = getattr(entry, "rating", None) - items.append( - { - "title": title, - "artist": "", - "album": kind, - "year": year, - "provider": self.name, - "imdb_id": imdb_id, - "kind": kind, - "rating": rating, - "raw": entry, - } - ) - return items - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - imdb_id = self._extract_imdb_id( - item.get("imdb_id") or item.get("id") or item.get("imdb") or "" - ) - try: - if imdb_id: - data = imdb_tag(imdb_id) - raw_tags = data.get("tag") if isinstance(data, dict) else [] - tags = [t for t in raw_tags if isinstance(t, str)] - if tags: - return tags - except Exception as exc: - log(f"IMDb tag extraction failed: {exc}", file=sys.stderr) - - tags = super().to_tags(item) - if imdb_id: - tags.append(f"imdb:{imdb_id}") - seen: set[str] = set() - deduped: List[str] = [] - for t in tags: - s = str(t or "").strip() - if not s: - continue - k = s.lower() - if k in seen: - continue - seen.add(k) - deduped.append(s) - return deduped - - def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: - return identifiers.get("imdb") - - -class YtdlpMetadataProvider(MetadataProvider): - """Metadata provider that extracts tags from a supported URL using yt-dlp. - - This does NOT download media; it only probes metadata. - """ - - @property - def name(self) -> str: # type: ignore[override] - return "ytdlp" - - def _extract_info(self, url: str) -> Optional[Dict[str, Any]]: - url = (url or "").strip() - if not url: - return None - - # Prefer Python module when available. - if yt_dlp is not None: - try: - opts: Any = { - "quiet": True, - "no_warnings": True, - "skip_download": True, - "noprogress": True, - "socket_timeout": 15, - "retries": 1, - "playlist_items": "1-10", - } - with yt_dlp.YoutubeDL(opts) as ydl: # type: ignore[attr-defined] - info = ydl.extract_info(url, download=False) - return cast(Dict[str, Any], info) if isinstance(info, dict) else None - except Exception: - pass - - # Fallback to CLI. - try: - cmd = [ - "yt-dlp", - "-J", - "--no-warnings", - "--skip-download", - "--playlist-items", - "1-10", - url, - ] - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - if proc.returncode != 0: - return None - payload = (proc.stdout or "").strip() - if not payload: - return None - data = json.loads(payload) - return data if isinstance(data, dict) else None - except Exception: - return None - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - url = (query or "").strip() - if not url.startswith(("http://", "https://")): - return [] - - info = self._extract_info(url) - if not isinstance(info, dict): - return [] - - upload_date = str(info.get("upload_date") or "") - release_date = str(info.get("release_date") or "") - year = (release_date - or upload_date)[:4] if (release_date or upload_date) else "" - - # Provide basic columns for the standard metadata selection table. - # NOTE: This is best-effort; many extractors don't provide artist/album. - artist = info.get("artist") or info.get("uploader") or info.get("channel") or "" - album = info.get("album") or info.get("playlist_title") or "" - title = info.get("title") or "" - - return [ - { - "title": title, - "artist": str(artist or ""), - "album": str(album or ""), - "year": str(year or ""), - "provider": self.name, - "url": url, - "raw": info, - } - ] - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - raw = item.get("raw") - if not isinstance(raw, dict): - return super().to_tags(item) - - tags: List[str] = [] - try: - from SYS.yt_metadata import extract_ytdlp_tags - except Exception: - extract_ytdlp_tags = None # type: ignore[assignment] - - if extract_ytdlp_tags: - try: - tags.extend(extract_ytdlp_tags(raw)) - except Exception: - pass - - # Subtitle availability tags - def _langs(value: Any) -> List[str]: - if not isinstance(value, dict): - return [] - out: List[str] = [] - for k in value.keys(): - if isinstance(k, str) and k.strip(): - out.append(k.strip().lower()) - return sorted(set(out)) - - # If this is a playlist container, subtitle/captions are usually per-entry. - info_for_subs: Dict[str, Any] = raw - entries = raw.get("entries") - if isinstance(entries, list) and entries: - first = entries[0] - if isinstance(first, dict): - info_for_subs = first - - for lang in _langs(info_for_subs.get("subtitles")): - tags.append(f"subs:{lang}") - for lang in _langs(info_for_subs.get("automatic_captions")): - tags.append(f"subs_auto:{lang}") - - # Always include source tag for parity with other providers. - tags.append(f"source:{self.name}") - - # Dedup case-insensitively, preserve order. - seen = set() - out: List[str] = [] - for t in tags: - if not isinstance(t, str): - continue - s = t.strip() - if not s: - continue - k = s.lower() - if k in seen: - continue - seen.add(k) - out.append(s) - return out - - def extract_url_query(self, result: Any, get_field: Any) -> Optional[str]: - raw_url = ( - get_field(result, "url", None) - or get_field(result, "source_url", None) - or get_field(result, "target", None) - ) - if isinstance(raw_url, list) and raw_url: - raw_url = raw_url[0] - if isinstance(raw_url, str): - text = raw_url.strip() - if text.startswith(("http://", "https://")): - return text - return None - - def emits_direct_tags(self) -> bool: - return True - - def default_subject_scrape_priority(self) -> int: - return 100 - - def url_scrape_priority(self, url: str) -> int: - text = str(url or "").strip() - if not text.startswith(("http://", "https://")): - return 0 - return 100 - - def prefers_store_tag_overwrite(self) -> bool: - return True - - def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: - return _dedup_text_values(tags) - - def _resolve_candidate_urls_for_subject( - self, - result: Any, - get_field: Any, - *, - backend: Any = None, - file_hash: Optional[str] = None, - ) -> List[str]: - try: - from SYS.metadata import normalize_urls - except Exception: - normalize_urls = None # type: ignore[assignment] - - urls: List[str] = [] - - if backend is not None and file_hash: - try: - backend_urls = backend.get_url(file_hash, config=self.config) - if backend_urls: - if normalize_urls: - urls.extend(normalize_urls(backend_urls)) - else: - urls.extend( - [str(u).strip() for u in backend_urls if isinstance(u, str) and str(u).strip()] - ) - except Exception: - pass - - try: - meta = backend.get_metadata(file_hash, config=self.config) - if isinstance(meta, dict) and meta.get("url"): - raw = meta.get("url") - if normalize_urls: - urls.extend(normalize_urls(raw)) - elif isinstance(raw, list): - urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) - elif isinstance(raw, str) and raw.strip(): - urls.append(raw.strip()) - except Exception: - pass - - for key in ("url", "webpage_url", "source_url", "target"): - val = get_field(result, key, None) - if not val: - continue - if normalize_urls: - urls.extend(normalize_urls(val)) - continue - if isinstance(val, str) and val.strip(): - urls.append(val.strip()) - elif isinstance(val, list): - urls.extend([str(u).strip() for u in val if isinstance(u, str) and str(u).strip()]) - - meta_field = get_field(result, "metadata", None) - if isinstance(meta_field, dict) and meta_field.get("url"): - raw = meta_field.get("url") - if normalize_urls: - urls.extend(normalize_urls(raw)) - elif isinstance(raw, list): - urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) - elif isinstance(raw, str) and raw.strip(): - urls.append(raw.strip()) - - return _dedup_text_values(urls) - - def _pick_supported_subject_url(self, urls: List[str]) -> Optional[str]: - if not urls: - return None - - def _is_hydrus_file_url(u: str) -> bool: - text = str(u or "").strip().lower() - return bool(text and "/get_files/file" in text and "hash=" in text) - - candidates = [] - for url in urls: - text = str(url or "").strip() - if not text.startswith(("http://", "https://")): - continue - if _is_hydrus_file_url(text): - continue - candidates.append(text) - if not candidates: - return None - - try: - from tool.ytdlp import is_url_supported_by_ytdlp - - for text in candidates: - try: - if is_url_supported_by_ytdlp(text): - return text - except Exception: - continue - except Exception: - pass - - return candidates[0] if candidates else None - - def resolve_subject_query( - self, - result: Any, - get_field: Any, - *, - backend: Any = None, - file_hash: Optional[str] = None, - ) -> Optional[str]: - candidate_urls = self._resolve_candidate_urls_for_subject( - result, - get_field, - backend=backend, - file_hash=file_hash, - ) - return self._pick_supported_subject_url(candidate_urls) - - @staticmethod - def _extract_url_formats(formats: Any) -> List[tuple[str, str]]: - if not isinstance(formats, list): - return [] - - video_formats: Dict[str, Dict[str, Any]] = {} - audio_formats: Dict[str, Dict[str, Any]] = {} - - for fmt in formats: - if not isinstance(fmt, dict): - continue - vcodec = fmt.get("vcodec", "none") - acodec = fmt.get("acodec", "none") - height = fmt.get("height") - ext = fmt.get("ext", "unknown") - format_id = fmt.get("format_id", "") - tbr = fmt.get("tbr", 0) - abr = fmt.get("abr", 0) - - if vcodec and vcodec != "none" and height: - if int(height) < 480: - continue - res_key = f"{int(height)}p" - if res_key not in video_formats or tbr > video_formats[res_key].get("tbr", 0): - video_formats[res_key] = { - "label": f"{int(height)}p ({ext})", - "format_id": str(format_id), - "tbr": tbr, - } - elif acodec and acodec != "none" and (not vcodec or vcodec == "none"): - audio_key = f"audio_{abr}" - if audio_key not in audio_formats or abr > audio_formats[audio_key].get("abr", 0): - audio_formats[audio_key] = { - "label": f"audio ({ext})", - "format_id": str(format_id), - "abr": abr, - } - - result: List[tuple[str, str]] = [] - for res in sorted(video_formats.keys(), key=lambda value: int(value.replace("p", "")), reverse=True): - fmt = video_formats[res] - result.append((str(fmt.get("label") or res), str(fmt.get("format_id") or ""))) - if audio_formats: - best_audio_key = max(audio_formats.keys(), key=lambda key: float(audio_formats[key].get("abr", 0) or 0)) - fmt = audio_formats[best_audio_key] - result.append((str(fmt.get("label") or "audio"), str(fmt.get("format_id") or ""))) - return [entry for entry in result if entry[1]] - - @staticmethod - def _build_playlist_items(raw: Dict[str, Any]) -> List[Dict[str, Any]]: - entries = raw.get("entries") - if not isinstance(entries, list): - return [] - - playlist_items: List[Dict[str, Any]] = [] - for idx, entry in enumerate(entries, 1): - if not isinstance(entry, dict): - continue - playlist_items.append( - { - "index": idx, - "id": entry.get("id", f"track_{idx}"), - "title": entry.get("title", entry.get("id", f"Track {idx}")), - "duration": entry.get("duration", 0), - "url": entry.get("url") or entry.get("webpage_url", ""), - } - ) - return playlist_items - - def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: - info = self._extract_info(url) - if not isinstance(info, dict): - return None - - item = { - "title": info.get("title") or "", - "artist": str(info.get("artist") or info.get("uploader") or info.get("channel") or ""), - "album": str(info.get("album") or info.get("playlist_title") or ""), - "year": str((str(info.get("release_date") or "") or str(info.get("upload_date") or ""))[:4]), - "provider": self.name, - "url": str(url or "").strip(), - "raw": info, - } - tags = _dedup_text_values([str(tag) for tag in self.to_tags(item) if tag is not None]) - return { - "title": item.get("title") or None, - "tag": tags, - "formats": self._extract_url_formats(info.get("formats", [])), - "playlist_items": self._build_playlist_items(info), - } - - -def _coerce_archive_field_list(value: Any) -> List[str]: - """Coerce an Archive.org metadata field to a list of strings.""" - - if value is None: - return [] - if isinstance(value, list): - out: List[str] = [] - for v in value: - try: - s = str(v).strip() - except Exception: - continue - if s: - out.append(s) - return out - if isinstance(value, (tuple, set)): - out = [] - for v in value: - try: - s = str(v).strip() - except Exception: - continue - if s: - out.append(s) - return out - try: - s = str(value).strip() - except Exception: - return [] - return [s] if s else [] - - -def archive_item_metadata_to_tags(archive_id: str, - item_metadata: Dict[str, Any]) -> List[str]: - """Coerce Archive.org metadata into a stable set of bibliographic tags.""" - - archive_id_clean = str(archive_id or "").strip() - meta = item_metadata if isinstance(item_metadata, dict) else {} - - tags: List[str] = [] - seen: set[str] = set() - - def _add(tag: str) -> None: - try: - t = str(tag).strip() - except Exception: - return - if not t: - return - if t.lower() in seen: - return - seen.add(t.lower()) - tags.append(t) - - if archive_id_clean: - _add(f"internet_archive:{archive_id_clean}") - - for title in _coerce_archive_field_list(meta.get("title"))[:1]: - _add(f"title:{title}") - - creators: List[str] = [] - creators.extend(_coerce_archive_field_list(meta.get("creator"))) - creators.extend(_coerce_archive_field_list(meta.get("author"))) - for creator in creators[:3]: - _add(f"author:{creator}") - - for publisher in _coerce_archive_field_list(meta.get("publisher"))[:3]: - _add(f"publisher:{publisher}") - - for date_val in _coerce_archive_field_list(meta.get("date"))[:1]: - _add(f"publish_date:{date_val}") - for year_val in _coerce_archive_field_list(meta.get("year"))[:1]: - _add(f"publish_date:{year_val}") - - for lang in _coerce_archive_field_list(meta.get("language"))[:3]: - _add(f"language:{lang}") - - for subj in _coerce_archive_field_list(meta.get("subject"))[:15]: - if len(subj) > 200: - subj = subj[:200] - _add(subj) - - def _clean_isbn(raw: str) -> str: - return str(raw or "").replace("-", "").strip() - - for isbn in _coerce_archive_field_list(meta.get("isbn"))[:10]: - isbn_clean = _clean_isbn(isbn) - if isbn_clean: - _add(f"isbn:{isbn_clean}") - - identifiers: List[str] = [] - identifiers.extend(_coerce_archive_field_list(meta.get("identifier"))) - identifiers.extend(_coerce_archive_field_list(meta.get("external-identifier"))) - added_other = 0 - for ident in identifiers: - ident_s = str(ident or "").strip() - if not ident_s: - continue - low = ident_s.lower() - - if low.startswith("urn:isbn:"): - val = _clean_isbn(ident_s.split(":", 2)[-1]) - if val: - _add(f"isbn:{val}") - continue - if low.startswith("isbn:"): - val = _clean_isbn(ident_s.split(":", 1)[-1]) - if val: - _add(f"isbn:{val}") - continue - if low.startswith("urn:oclc:"): - val = ident_s.split(":", 2)[-1].strip() - if val: - _add(f"oclc:{val}") - continue - if low.startswith("oclc:"): - val = ident_s.split(":", 1)[-1].strip() - if val: - _add(f"oclc:{val}") - continue - if low.startswith("urn:lccn:"): - val = ident_s.split(":", 2)[-1].strip() - if val: - _add(f"lccn:{val}") - continue - if low.startswith("lccn:"): - val = ident_s.split(":", 1)[-1].strip() - if val: - _add(f"lccn:{val}") - continue - if low.startswith("doi:"): - val = ident_s.split(":", 1)[-1].strip() - if val: - _add(f"doi:{val}") - continue - - if archive_id_clean and low == archive_id_clean.lower(): - continue - if added_other >= 5: - continue - if len(ident_s) > 200: - ident_s = ident_s[:200] - _add(f"identifier:{ident_s}") - added_other += 1 - - return tags - - -def fetch_archive_item_metadata(archive_id: str, - *, - timeout: int = 8) -> Dict[str, Any]: - ident = str(archive_id or "").strip() - if not ident: - return {} - resp = get_requests_session().get( - f"https://archive.org/metadata/{ident}", - timeout=int(timeout), - ) - resp.raise_for_status() - data = resp.json() if resp is not None else {} - if not isinstance(data, dict): - return {} - meta = data.get("metadata") - return meta if isinstance(meta, dict) else {} - - -def scrape_isbn_metadata(isbn: str) -> List[str]: - """Scrape metadata tags for an ISBN using OpenLibrary's books API.""" - - new_tags: List[str] = [] - - isbn_clean = str(isbn or "").replace("isbn:", "").replace("-", "").strip() - if not isbn_clean: - return [] - - url = f"https://openlibrary.org/api/books?bibkeys=ISBN:{isbn_clean}&jscmd=data&format=json" - try: - with HTTPClient() as client: - response = client.get(url) - response.raise_for_status() - data = json.loads(response.content.decode("utf-8")) - except Exception as exc: - log(f"Failed to fetch ISBN metadata: {exc}", file=sys.stderr) - return [] - - if not data: - log(f"No ISBN metadata found for: {isbn}") - return [] - - book_data = next(iter(data.values()), None) - if not isinstance(book_data, dict): - return [] - - if "title" in book_data: - new_tags.append(f"title:{book_data['title']}") - - authors = book_data.get("authors") - if isinstance(authors, list): - for author in authors[:3]: - if isinstance(author, dict) and author.get("name"): - new_tags.append(f"author:{author['name']}") - - if book_data.get("publish_date"): - new_tags.append(f"publish_date:{book_data['publish_date']}") - - publishers = book_data.get("publishers") - if isinstance(publishers, list) and publishers: - pub = publishers[0] - if isinstance(pub, dict) and pub.get("name"): - new_tags.append(f"publisher:{pub['name']}") - - if "description" in book_data: - desc = book_data.get("description") - if isinstance(desc, dict) and "value" in desc: - desc = desc.get("value") - if desc: - desc_str = str(desc).strip() - if desc_str: - new_tags.append(f"description:{desc_str[:200]}") - - page_count = book_data.get("number_of_pages") - if isinstance(page_count, int) and page_count > 0: - new_tags.append(f"pages:{page_count}") - - identifiers = book_data.get("identifiers") - if isinstance(identifiers, dict): - - def _first(value: Any) -> Any: - if isinstance(value, list) and value: - return value[0] - return value - - for key, ns in ( - ("openlibrary", "openlibrary"), - ("lccn", "lccn"), - ("oclc", "oclc"), - ("goodreads", "goodreads"), - ("librarything", "librarything"), - ("doi", "doi"), - ("internet_archive", "internet_archive"), - ): - val = _first(identifiers.get(key)) - if val: - new_tags.append(f"{ns}:{val}") - - debug(f"Found {len(new_tags)} tag(s) from ISBN lookup") - return new_tags - - -def scrape_openlibrary_metadata(olid: str) -> List[str]: - """Scrape metadata tags for an OpenLibrary ID using the edition JSON endpoint.""" - - new_tags: List[str] = [] - - olid_text = str(olid or "").strip() - if not olid_text: - return [] - - olid_norm = olid_text - try: - if not olid_norm.startswith("OL"): - olid_norm = f"OL{olid_norm}" - if not olid_norm.endswith("M"): - olid_norm = f"{olid_norm}M" - except Exception: - olid_norm = olid_text - - new_tags.append(f"openlibrary:{olid_norm}") - - olid_clean = olid_text.replace("OL", "").replace("M", "") - if not olid_clean.isdigit(): - olid_clean = olid_text - - if not olid_text.startswith("OL"): - url = f"https://openlibrary.org/books/OL{olid_clean}M.json" - else: - url = f"https://openlibrary.org/books/{olid_text}.json" - - try: - with HTTPClient() as client: - response = client.get(url) - response.raise_for_status() - data = json.loads(response.content.decode("utf-8")) - except Exception as exc: - log(f"Failed to fetch OpenLibrary metadata: {exc}", file=sys.stderr) - return [] - - if not isinstance(data, dict) or not data: - log(f"No OpenLibrary metadata found for: {olid_text}") - return [] - - if "title" in data: - new_tags.append(f"title:{data['title']}") - - authors = data.get("authors") - if isinstance(authors, list): - for author in authors[:3]: - if isinstance(author, dict) and author.get("name"): - new_tags.append(f"author:{author['name']}") - continue - - author_key = None - if isinstance(author, dict): - if isinstance(author.get("author"), dict): - author_key = author.get("author", {}).get("key") - if not author_key: - author_key = author.get("key") - - if isinstance(author_key, str) and author_key.startswith("/"): - try: - author_url = f"https://openlibrary.org{author_key}.json" - with HTTPClient(timeout=10) as client: - author_resp = client.get(author_url) - author_resp.raise_for_status() - author_data = json.loads(author_resp.content.decode("utf-8")) - if isinstance(author_data, dict) and author_data.get("name"): - new_tags.append(f"author:{author_data['name']}") - continue - except Exception: - pass - - if isinstance(author, str) and author: - new_tags.append(f"author:{author}") - - if data.get("publish_date"): - new_tags.append(f"publish_date:{data['publish_date']}") - - publishers = data.get("publishers") - if isinstance(publishers, list) and publishers: - pub = publishers[0] - if isinstance(pub, dict) and pub.get("name"): - new_tags.append(f"publisher:{pub['name']}") - elif isinstance(pub, str) and pub: - new_tags.append(f"publisher:{pub}") - - if "description" in data: - desc = data.get("description") - if isinstance(desc, dict) and "value" in desc: - desc = desc.get("value") - if desc: - desc_str = str(desc).strip() - if desc_str: - new_tags.append(f"description:{desc_str[:200]}") - - page_count = data.get("number_of_pages") - if isinstance(page_count, int) and page_count > 0: - new_tags.append(f"pages:{page_count}") - - subjects = data.get("subjects") - if isinstance(subjects, list): - for subject in subjects[:10]: - if isinstance(subject, str): - subject_clean = subject.strip() - if subject_clean and subject_clean not in new_tags: - new_tags.append(subject_clean) - - identifiers = data.get("identifiers") - if isinstance(identifiers, dict): - - def _first(value: Any) -> Any: - if isinstance(value, list) and value: - return value[0] - return value - - for key, ns in ( - ("isbn_10", "isbn_10"), - ("isbn_13", "isbn_13"), - ("lccn", "lccn"), - ("oclc_numbers", "oclc"), - ("goodreads", "goodreads"), - ("internet_archive", "internet_archive"), - ): - val = _first(identifiers.get(key)) - if val: - new_tags.append(f"{ns}:{val}") - - ocaid = data.get("ocaid") - if isinstance(ocaid, str) and ocaid.strip(): - new_tags.append(f"internet_archive:{ocaid.strip()}") - - debug(f"Found {len(new_tags)} tag(s) from OpenLibrary lookup") - return new_tags - - -SAMPLE_ITEMS: List[Dict[str, Any]] = [ - { - "title": "Sample OpenLibrary book", - "path": "https://openlibrary.org/books/OL123M", - "openlibrary_id": "OL123M", - "archive_id": "samplearchive123", - "availability": "borrow", - "availability_reason": "sample", - "direct_url": "https://archive.org/download/sample.pdf", - "author_name": ["OpenLibrary Demo"], - "first_publish_year": 2023, - "ia": ["samplearchive123"], - }, -] - - -try: - from typing import Iterable - - from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column - from SYS.result_table_adapters import register_plugin - - def _ensure_search_result(item: Any) -> SearchResult: - if isinstance(item, SearchResult): - return item - if isinstance(item, dict): - data = dict(item) - title = str(data.get("title") or data.get("name") or "OpenLibrary") - path = str(data.get("path") or data.get("url") or "") - detail = str(data.get("detail") or "") - annotations = list(data.get("annotations") or []) - media_kind = str(data.get("media_kind") or "book") - return SearchResult( - table="openlibrary", - title=title, - path=path, - detail=detail, - annotations=annotations, - media_kind=media_kind, - columns=data.get("columns") or [], - full_metadata={**data, "raw": dict(item)}, - ) - return SearchResult( - table="openlibrary", - title=str(item or "OpenLibrary"), - path="", - detail="", - annotations=[], - media_kind="book", - full_metadata={"raw": {}}, - ) - - def _adapter(items: Iterable[Any]) -> Iterable[ResultModel]: - for item in items: - sr = _ensure_search_result(item) - metadata = dict(getattr(sr, "full_metadata", {}) or {}) - raw = metadata.get("raw") - if isinstance(raw, dict): - normalized = normalize_record(raw) - for key, val in normalized.items(): - metadata.setdefault(key, val) - - def _make_url() -> str: - candidate = ( - metadata.get("selection_url") or - metadata.get("direct_url") or - metadata.get("url") or - metadata.get("path") or - sr.path or - "" - ) - return str(candidate or "").strip() - - selection_url = _make_url() - if selection_url: - metadata["selection_url"] = selection_url - authors_value = metadata.get("authors_display") or metadata.get("authors") or metadata.get("author_name") or "" - if isinstance(authors_value, list): - authors_value = ", ".join(str(v) for v in authors_value if v) - authors_text = str(authors_value or "").strip() - if authors_text: - metadata["authors_display"] = authors_text - year_value = metadata.get("year") or metadata.get("first_publish_year") - if year_value and not isinstance(year_value, str): - year_value = str(year_value) - if year_value: - metadata["year"] = str(year_value) - metadata.setdefault("openlibrary_id", metadata.get("openlibrary_id") or metadata.get("olid")) - metadata.setdefault("source", metadata.get("source") or "openlibrary") - yield ResultModel( - title=str(sr.title or metadata.get("title") or selection_url or "OpenLibrary"), - path=selection_url or None, - metadata=metadata, - source="openlibrary", - ) - - def _columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: - cols: List[ColumnSpec] = [title_column()] - def _has(key: str) -> bool: - return any((row.metadata or {}).get(key) for row in rows) - - if _has("authors_display"): - cols.append( - ColumnSpec( - "authors_display", - "Author", - lambda r: (r.metadata or {}).get("authors_display") or "", - ) - ) - if _has("year"): - cols.append(metadata_column("year", "Year")) - if _has("availability"): - cols.append(metadata_column("availability", "Avail")) - if _has("archive_id"): - cols.append(metadata_column("archive_id", "Archive ID")) - if _has("openlibrary_id"): - cols.append(metadata_column("openlibrary_id", "OLID")) - return cols - - def _selection_fn(row: ResultModel) -> List[str]: - metadata = row.metadata or {} - url = str(metadata.get("selection_url") or row.path or "").strip() - if url: - return ["-url", url] - return ["-title", row.title or ""] - - register_plugin( - "openlibrary", - _adapter, - columns=_columns_factory, - selection_fn=_selection_fn, - metadata={"description": "OpenLibrary search provider (JSON result table template)"}, - ) -except Exception: - pass - - -# Registry --------------------------------------------------------------- - -class TidalMetadataProvider(MetadataProvider): - """Metadata provider that reuses the Tidal search provider for tidal info.""" - - @property - def name(self) -> str: # type: ignore[override] - return "tidal" - - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: - if Tidal is None: - raise RuntimeError("Tidal provider unavailable for tidal metadata") - super().__init__(config) - self._provider = Tidal(self.config) - - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: - normalized = str(query or "").strip() - if not normalized: - return [] - - try: - results = self._provider.search(normalized, limit=limit) - except Exception as exc: - debug(f"[tidal-meta] search failed for '{normalized}': {exc}") - return [] - - items: List[Dict[str, Any]] = [] - for result in results: - metadata = getattr(result, "full_metadata", {}) or {} - if not isinstance(metadata, dict): - metadata = {} - - title = stringify(metadata.get("title") or result.title) - if not title: - continue - - artists = extract_artists(metadata) - artist_display = ", ".join(artists) if artists else stringify(metadata.get("artist")) - - album_obj = metadata.get("album") - album = "" - if isinstance(album_obj, dict): - album = stringify(album_obj.get("title")) - else: - album = stringify(metadata.get("album")) - - year = stringify(metadata.get("releaseDate") or metadata.get("year") or metadata.get("date")) - - track_id = self._provider._parse_track_id(metadata.get("trackId") or metadata.get("id")) - lyrics_data = None - if track_id is not None: - try: - lyrics_data = self._provider._fetch_track_lyrics(track_id) - except Exception as exc: - debug(f"[tidal-meta] lyrics lookup failed for {track_id}: {exc}") - - lyrics = None - if isinstance(lyrics_data, dict): - lyrics = stringify(lyrics_data.get("lyrics") or lyrics_data.get("text")) - subtitles = stringify(lyrics_data.get("subtitles")) - if subtitles: - metadata.setdefault("_tidal_lyrics", {})["subtitles"] = subtitles - - tags = sorted(build_track_tags(metadata)) - items.append({ - "title": title, - "artist": artist_display, - "album": album, - "year": year, - "lyrics": lyrics, - "tags": tags, - "provider": self.name, - "path": getattr(result, "path", ""), - "track_id": track_id, - "full_metadata": metadata, - }) - return items - - def to_tags(self, item: Dict[str, Any]) -> List[str]: - tags: List[str] = [] - for value in item.get("tags", []): - value_text = stringify(value) - if value_text: - normalized = value_text.lower() - if normalized in {"tidal", "lossless"}: - continue - if normalized.startswith("quality:lossless"): - continue - tags.append(value_text) - return tags - -_METADATA_PROVIDERS: Dict[str, - Type[MetadataProvider]] = { - "itunes": ITunesProvider, - "openlibrary": OpenLibraryMetadataProvider, - "googlebooks": GoogleBooksMetadataProvider, - "google": GoogleBooksMetadataProvider, - "isbnsearch": ISBNsearchMetadataProvider, - "musicbrainz": MusicBrainzMetadataProvider, - "imdb": ImdbMetadataProvider, - "ytdlp": YtdlpMetadataProvider, - "tidal": TidalMetadataProvider, - } - - -def register_provider(name: str, provider_cls: Type[MetadataProvider]) -> None: - _METADATA_PROVIDERS[name.lower()] = provider_cls - - -def list_metadata_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: - availability: Dict[str, - bool] = {} - for name, cls in _METADATA_PROVIDERS.items(): - try: - _ = cls(config) - # Basic availability check: perform lightweight validation if defined - availability[name] = True - except Exception: - availability[name] = False - return availability - - -def get_metadata_provider(name: str, - config: Optional[Dict[str, - Any]] = None - ) -> Optional[MetadataProvider]: - cls = _METADATA_PROVIDERS.get(name.lower()) - if not cls: - return None - try: - return cls(config) - except Exception as exc: - log(f"Provider init failed for '{name}': {exc}", file=sys.stderr) - return None - - -def get_default_subject_scrape_provider( - config: Optional[Dict[str, Any]] = None, -) -> Optional[MetadataProvider]: - best_provider: Optional[MetadataProvider] = None - best_priority = 0 - for cls in _METADATA_PROVIDERS.values(): - try: - provider = cls(config) - priority = int(provider.default_subject_scrape_priority()) - except Exception: - continue - if priority > best_priority: - best_priority = priority - best_provider = provider - return best_provider - - -def get_metadata_provider_for_url( - url: str, - config: Optional[Dict[str, Any]] = None, -) -> Optional[MetadataProvider]: - text = str(url or "").strip() - if not text: - return None - - best_provider: Optional[MetadataProvider] = None - best_priority = 0 - for cls in _METADATA_PROVIDERS.values(): - try: - provider = cls(config) - priority = int(provider.url_scrape_priority(text)) - except Exception: - continue - if priority > best_priority: - best_priority = priority - best_provider = provider - return best_provider +from plugins.metadata_provider import * # noqa: F401,F403 diff --git a/Provider/tidal_manifest.py b/Provider/tidal_manifest.py index 00dcb1e..0ae9148 100644 --- a/Provider/tidal_manifest.py +++ b/Provider/tidal_manifest.py @@ -1,343 +1,8 @@ -"""Tidal/HIFI manifest helpers. +"""Legacy compatibility shim for Tidal manifest helpers. -This module intentionally lives with the provider code (not cmdlets). -It contains best-effort helpers for turning proxy-provided Tidal "manifest" -values into a playable input reference: -- A local MPD file path (persisted to temp) -- Or a direct URL (when the manifest is JSON with `urls`) - -Callers may pass either a SearchResult-like object (with `.full_metadata`) or -pipeline dicts. +The active implementation now lives in ``plugins.tidal_manifest`` so the +plugin namespace owns the manifest helper module. Keep this file only to avoid +breaking old imports while the legacy ``Provider`` package is phased out. """ -from __future__ import annotations - -import base64 -import hashlib -import json -import re -import sys -import tempfile -from pathlib import Path -from typing import Any, Dict, Optional - -from API.httpx_shared import get_shared_httpx_client -from SYS.logger import log - - -_DEFAULT_TIDAL_TRACK_API_BASES = ( - "https://triton.squid.wtf", - "https://wolf.qqdl.site", - "https://maus.qqdl.site", - "https://vogel.qqdl.site", - "https://katze.qqdl.site", - "https://hund.qqdl.site", - "https://tidal.kinoplus.online", - "https://tidal-api.binimum.org", -) - - -def resolve_tidal_manifest_path(item: Any) -> Optional[str]: - """Persist the Tidal manifest (MPD) and return a local path or URL. - - Resolution order: - 1) `_tidal_manifest_path` (existing local file) - 2) `_tidal_manifest_url` (existing remote URL) - 3) decode `manifest` and: - - if JSON with `urls`: return the first URL - - if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path - - If `manifest` is missing but a track id exists, the function will attempt a - best-effort fetch from the public proxy endpoints to populate `manifest`. - """ - - metadata: Any = None - if isinstance(item, dict): - metadata = item.get("full_metadata") or item.get("metadata") - else: - metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None) - - if not isinstance(metadata, dict): - return None - - existing_path = metadata.get("_tidal_manifest_path") - if existing_path: - try: - resolved = Path(str(existing_path)) - if resolved.is_file(): - return str(resolved) - except Exception: - pass - - existing_url = metadata.get("_tidal_manifest_url") - if existing_url and isinstance(existing_url, str): - candidate = existing_url.strip() - if candidate: - return candidate - - raw_manifest = metadata.get("manifest") - if not raw_manifest: - _maybe_fetch_track_manifest(item, metadata) - raw_manifest = metadata.get("manifest") - if not raw_manifest: - return None - - manifest_str = "".join(str(raw_manifest or "").split()) - if not manifest_str: - return None - - manifest_bytes: bytes - try: - manifest_bytes = base64.b64decode(manifest_str, validate=True) - except Exception: - try: - manifest_bytes = base64.b64decode(manifest_str, validate=False) - except Exception: - try: - manifest_bytes = manifest_str.encode("utf-8") - except Exception: - return None - - if not manifest_bytes: - return None - - head = (manifest_bytes[:1024] or b"").lstrip() - if head.startswith((b"{", b"[")): - return _resolve_json_manifest_urls(metadata, manifest_bytes) - - looks_like_mpd = head.startswith((b" Optional[str]: - text = str(candidate or "").strip() - if not text: - return None - if not re.match(r"^https?://", text, flags=re.IGNORECASE): - return None - return text.rstrip("/") - - -def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]: - bases: list[str] = [] - seen: set[str] = set() - - dynamic_candidates = [ - metadata.get("_tidal_api_base"), - metadata.get("_api_base"), - metadata.get("api_base"), - metadata.get("base_url"), - ] - - for candidate in dynamic_candidates: - normalized = _normalize_api_base(candidate) - if normalized and normalized not in seen: - seen.add(normalized) - bases.append(normalized) - - for candidate in _DEFAULT_TIDAL_TRACK_API_BASES: - normalized = _normalize_api_base(candidate) - if normalized and normalized not in seen: - seen.add(normalized) - bases.append(normalized) - - return bases - - -def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None: - """If we only have a track id, fetch details from the proxy to populate `manifest`.""" - - try: - already = bool(metadata.get("_tidal_track_details_fetched")) - except Exception: - already = False - - track_id = metadata.get("trackId") or metadata.get("id") - - if track_id is None: - try: - if isinstance(item, dict): - candidate_path = item.get("path") or item.get("url") - else: - candidate_path = getattr(item, "path", None) or getattr(item, "url", None) - except Exception: - candidate_path = None - - if candidate_path: - m = re.search( - r"(tidal|hifi):(?://)?track[\\/](\d+)", - str(candidate_path), - flags=re.IGNORECASE, - ) - if m: - track_id = m.group(2) - - if already or track_id is None: - return - - try: - track_int = int(track_id) - except Exception: - track_int = None - - if not track_int or track_int <= 0: - return - - try: - client = get_shared_httpx_client() - except Exception: - return - - attempted = False - for base in _iter_track_api_bases(metadata): - attempted = True - - track_data: Optional[Dict[str, Any]] = None - for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}): - try: - resp = client.get( - f"{base}/track/", - params=params, - timeout=10.0, - ) - resp.raise_for_status() - payload = resp.json() - data = payload.get("data") if isinstance(payload, dict) else None - if isinstance(data, dict) and data: - track_data = data - break - except Exception: - continue - - if isinstance(track_data, dict) and track_data: - try: - metadata.update(track_data) - except Exception: - pass - - if not metadata.get("manifest") or not metadata.get("url"): - try: - resp_info = client.get( - f"{base}/info/", - params={"id": str(track_int)}, - timeout=10.0, - ) - resp_info.raise_for_status() - info_payload = resp_info.json() - info_data = info_payload.get("data") if isinstance(info_payload, dict) else None - if isinstance(info_data, dict) and info_data: - try: - for key, value in info_data.items(): - if key not in metadata or not metadata.get(key): - metadata[key] = value - except Exception: - pass - except Exception: - pass - - if metadata.get("manifest"): - break - - if attempted: - try: - metadata["_tidal_track_details_fetched"] = True - except Exception: - pass - - -def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]: - try: - text = manifest_bytes.decode("utf-8", errors="ignore") - payload = json.loads(text) - urls = payload.get("urls") or [] - selected_url = None - for candidate in urls: - if isinstance(candidate, str): - candidate = candidate.strip() - if candidate: - selected_url = candidate - break - if selected_url: - try: - metadata["_tidal_manifest_url"] = selected_url - except Exception: - pass - return selected_url - try: - metadata["_tidal_manifest_error"] = "JSON manifest contained no urls" - except Exception: - pass - log( - f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls", - file=sys.stderr, - ) - except Exception as exc: - try: - metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}" - except Exception: - pass - log( - f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}", - file=sys.stderr, - ) - return None - - -def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]: - manifest_hash = str(metadata.get("manifestHash") or "").strip() - track_id = metadata.get("trackId") or metadata.get("id") - - identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest() - identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64] - if not identifier_safe: - identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12] - - track_safe = "tidal" - if track_id is not None: - track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal" - - manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal" - try: - manifest_dir.mkdir(parents=True, exist_ok=True) - except Exception: - pass - - filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd" - target_path = manifest_dir / filename - - try: - with open(target_path, "wb") as fh: - fh.write(manifest_bytes) - metadata["_tidal_manifest_path"] = str(target_path) - - # Best-effort: propagate back into the caller object/dict. - if isinstance(item, dict): - if item.get("full_metadata") is metadata: - item["full_metadata"] = metadata - elif item.get("metadata") is metadata: - item["metadata"] = metadata - else: - extra = getattr(item, "extra", None) - if isinstance(extra, dict): - extra["_tidal_manifest_path"] = str(target_path) - - return str(target_path) - except Exception: - return None +from plugins.tidal_manifest import * # noqa: F401,F403 diff --git a/ProviderCore/base.py b/ProviderCore/base.py index dd099b6..cc9e05f 100644 --- a/ProviderCore/base.py +++ b/ProviderCore/base.py @@ -114,7 +114,7 @@ def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]: class Provider(ABC): """Unified plugin base class. - This replaces the older split between search and upload providers. + This replaces the older split between search and upload plugins. Concrete plugins may implement any subset of: - search(query, ...) - download(result, output_dir) @@ -127,8 +127,8 @@ class Provider(ABC): PLUGIN_NAME: str = "" PLUGIN_ALIASES: Sequence[str] = () - # Optional provider-driven defaults for what to do when a user selects @N from a - # provider table. The CLI uses this to auto-insert stages (e.g. download-file) + # Optional plugin-driven defaults for what to do when a user selects @N from a + # plugin table. The CLI uses this to auto-insert stages (e.g. download-file) # without hardcoding table names. # # Example: @@ -138,7 +138,7 @@ class Provider(ABC): TABLE_AUTO_PREFIXES: Dict[str, Sequence[str]] = {} AUTO_STAGE_USE_SELECTION_ARGS: bool = False - # Optional provider-declared configuration keys. + # Optional plugin-declared configuration keys. # Used for dynamically generating config panels (e.g., missing credentials). REQUIRED_CONFIG_KEYS: Sequence[str] = () @@ -176,7 +176,7 @@ class Provider(ABC): return False def get_table_type(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: - """Return the table type identifier for results from this provider.""" + """Return the table type identifier for results from this plugin.""" return self.name def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: @@ -197,12 +197,12 @@ class Provider(ABC): @property def prefers_transfer_progress(self) -> bool: - """True if this provider prefers explicit transfer progress tracking (begin/finish) during download.""" + """True if this plugin prefers explicit transfer progress tracking (begin/finish) during download.""" return False @classmethod def config_schema(cls) -> List[Dict[str, Any]]: - """Return configuration schema for this provider. + """Return configuration schema for this plugin. Returns a list of dicts, each defining a field: { @@ -234,8 +234,124 @@ class Provider(ABC): return [] return out + @classmethod + def plugin_config_key(cls) -> str: + return str(getattr(cls, "PLUGIN_NAME", None) or cls.__name__ or "").strip().lower() + + @classmethod + def plugin_instance_filter_keys(cls) -> Tuple[str, ...]: + return ("instance", "store") + + @classmethod + def plugin_config_field_keys(cls) -> set[str]: + keys: set[str] = set() + try: + for field in cls.config_schema() or []: + if not isinstance(field, dict): + continue + key = str(field.get("key") or "").strip().lower() + if key: + keys.add(key) + except Exception: + return set() + return keys + + def requested_instance_name( + self, + filters: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Optional[str]: + for key in self.plugin_instance_filter_keys(): + value = kwargs.get(key) + if value in (None, "") and isinstance(filters, dict): + value = filters.get(key) + text = str(value or "").strip() + if text: + return text + return None + + def plugin_config_root(self) -> Dict[str, Any]: + if not isinstance(self.config, dict): + return {} + provider_cfg = self.config.get("provider") + if not isinstance(provider_cfg, dict): + return {} + entry = provider_cfg.get(self.plugin_config_key()) + return dict(entry) if isinstance(entry, dict) else {} + + def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]: + entry = self.plugin_config_root() + if not entry: + return {} + + schema_keys = self.plugin_config_field_keys() + entry_keys = {str(key or "").strip().lower() for key in entry.keys()} + looks_like_single = bool(schema_keys and entry_keys.intersection(schema_keys)) + if not looks_like_single and entry: + looks_like_single = not all(isinstance(value, dict) for value in entry.values()) + + if looks_like_single: + return {"default": dict(entry)} + + instances: Dict[str, Dict[str, Any]] = {} + for raw_name, raw_value in entry.items(): + if not isinstance(raw_value, dict): + continue + name = str(raw_name or "").strip() + if not name: + continue + instances[name] = dict(raw_value) + return instances + + def configured_instances(self) -> List[str]: + instances = self.plugin_instance_configs() + if not instances: + return [] + if set(instances.keys()) == {"default"}: + return [] + return list(instances.keys()) + + def resolve_plugin_instance( + self, + instance_name: Optional[str] = None, + *, + require_explicit: bool = False, + ) -> Tuple[Optional[str], Dict[str, Any]]: + instances = self.plugin_instance_configs() + if not instances: + return None, {} + + requested = str(instance_name or "").strip() + if not requested: + first_name = next(iter(instances.keys())) + resolved = dict(instances[first_name]) + if first_name != "default": + resolved.setdefault("_instance_name", first_name) + return (None if first_name == "default" else first_name), resolved + + requested_lower = requested.lower() + for name, cfg in instances.items(): + aliases = {str(name).strip().lower()} + explicit_name = str(cfg.get("_instance_name") or cfg.get("instance") or cfg.get("name") or "").strip().lower() + if explicit_name: + aliases.add(explicit_name) + if requested_lower in aliases: + resolved = dict(cfg) + if name != "default": + resolved.setdefault("_instance_name", name) + return (None if name == "default" else name), resolved + + if require_explicit: + return None, {} + + first_name = next(iter(instances.keys())) + resolved = dict(instances[first_name]) + if first_name != "default": + resolved.setdefault("_instance_name", first_name) + return (None if first_name == "default" else first_name), resolved + def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: - """Allow providers to normalize query text and parse inline arguments.""" + """Allow plugins to normalize query text and parse inline arguments.""" normalized = str(query or "").strip() return normalized, {} @@ -250,9 +366,9 @@ class Provider(ABC): table_type: str = "", table_meta: Optional[Dict[str, Any]] = None, ) -> Tuple[List[SearchResult], Optional[str], Optional[Dict[str, Any]]]: - """Optional hook for provider-specific result transforms. + """Optional hook for plugin-specific result transforms. - Cmdlets should avoid hardcoding provider quirks. Providers can override + Cmdlets should avoid hardcoding plugin quirks. Plugins can override this to: - expand/replace result sets (e.g., artist -> albums) - override the table type @@ -282,7 +398,7 @@ class Provider(ABC): **kwargs: Any, ) -> List[SearchResult]: """Search for items matching the query.""" - raise NotImplementedError(f"Provider '{self.name}' does not support search") + raise NotImplementedError(f"Plugin '{self.name}' does not support search") def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: """Download an item from a search result.""" @@ -355,7 +471,7 @@ class Provider(ABC): } def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]: - """Optional provider override to parse and act on URLs.""" + """Optional plugin override to parse and act on URLs.""" _ = url _ = output_dir @@ -428,24 +544,24 @@ class Provider(ABC): return "" def config_actions(self) -> List[Dict[str, Any]]: - """Optional actions exposed in the config editor for this provider.""" + """Optional actions exposed in the config editor for this plugin.""" return [] def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]: - """Execute a provider-owned config action from the config editor.""" + """Execute a plugin-owned config action from the config editor.""" return { "ok": False, - "message": f"Provider '{self.name}' does not support config action '{action_id}'.", + "message": f"Plugin '{self.name}' does not support config action '{action_id}'.", } def upload(self, file_path: str, **kwargs: Any) -> str: """Upload a file and return a URL or identifier.""" - raise NotImplementedError(f"Provider '{self.name}' does not support upload") + raise NotImplementedError(f"Plugin '{self.name}' does not support upload") def validate(self) -> bool: - """Check if provider is available and properly configured.""" + """Check if the plugin is available and properly configured.""" return True @@ -459,7 +575,7 @@ class Provider(ABC): ) -> bool: """Optional hook for handling `@N` selection semantics. - The CLI can delegate selection behavior to a provider/store instead of + The CLI can delegate selection behavior to a plugin/store instead of applying the default selection filtering. Return True if the selection was handled and default behavior should be skipped. @@ -470,6 +586,101 @@ class Provider(ABC): _ = stage_is_last return False + def show_selection_details( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + source_command: str = "", + table_type: str = "", + table_metadata: Optional[Dict[str, Any]] = None, + **_kwargs: Any, + ) -> bool: + """Optionally render a terminal detail view for a selected plugin row.""" + + _selected_item, payload, _metadata = self.resolve_selection_detail_subject( + selected_items, + stage_is_last=stage_is_last, + source_command=source_command, + ) + _ = table_type + _ = table_metadata + if not isinstance(payload, dict): + return False + + detail_title = self.label + item_title = str(payload.get("title") or "").strip() + if item_title: + detail_title = f"{self.label}: {item_title}" + + try: + from SYS.rich_display import render_item_details_panel + + render_item_details_panel(payload, title=detail_title) + except Exception: + return False + + try: + if hasattr(ctx, "set_last_result_items_only"): + ctx.set_last_result_items_only([payload]) + except Exception: + pass + + return True + + def resolve_selection_detail_subject( + self, + selected_items: List[Any], + *, + stage_is_last: bool = True, + source_command: str = "", + require_media_kind: Optional[str] = None, + ) -> Tuple[Optional[Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: + """Normalize a terminal `@N` selection into `(item, payload, metadata)`. + + Custom plugin detail hooks can use this to share the common preconditions + for item panels instead of re-checking terminal/single-row/search-file + state in each plugin. + """ + + if not stage_is_last or len(selected_items or []) != 1: + return None, None, None + + normalized_source = str(source_command or "").replace("_", "-").strip().lower() + if normalized_source != "search-file": + return None, None, None + + item: Any = selected_items[0] + payload: Optional[Dict[str, Any]] + if isinstance(item, dict): + payload = item + else: + payload = None + to_dict = getattr(item, "to_dict", None) + if callable(to_dict): + try: + maybe = to_dict() + except Exception: + maybe = None + if isinstance(maybe, dict): + payload = maybe + + if not isinstance(payload, dict): + return item, None, None + + meta: Dict[str, Any] = {} + nested = payload.get("full_metadata") or payload.get("metadata") + if isinstance(nested, dict): + meta = nested + + if require_media_kind: + media_kind = str(payload.get("media_kind") or meta.get("media_kind") or "").strip().lower() + if media_kind != str(require_media_kind or "").strip().lower(): + return item, None, None + + return item, payload, meta + @classmethod def selection_auto_stage( cls, @@ -478,10 +689,10 @@ class Provider(ABC): ) -> Optional[List[str]]: """Return a stage to auto-run after selecting from `table_type`. - This is used by the CLI to auto-insert default stages for provider tables + This is used by the CLI to auto-insert default stages for plugin tables (e.g. select a YouTube row -> auto-run download-file). - Providers can implement this via class attributes (TABLE_AUTO_STAGES / + Plugins can implement this via class attributes (TABLE_AUTO_STAGES / TABLE_AUTO_PREFIXES) or by overriding this method. """ t = str(table_type or "").strip().lower() @@ -522,7 +733,7 @@ class Provider(ABC): @classmethod def url_patterns(cls) -> Tuple[str, ...]: - """Return normalized URL patterns that this provider handles.""" + """Return normalized URL patterns that this plugin handles.""" patterns: List[str] = [] maybe_urls = getattr(cls, "URL", None) if isinstance(maybe_urls, (list, tuple)): @@ -564,15 +775,3 @@ class Provider(ABC): return tuple(prefixes) -class SearchProvider(Provider): - """Compatibility alias for older code. - - Prefer inheriting from Provider directly. - """ - - -class FileProvider(Provider): - """Compatibility alias for older code. - - Prefer inheriting from Provider directly. - """ diff --git a/ProviderCore/registry.py b/ProviderCore/registry.py index 1a510b4..eda0acc 100644 --- a/ProviderCore/registry.py +++ b/ProviderCore/registry.py @@ -22,12 +22,24 @@ from urllib.parse import urlparse from SYS.logger import log, debug -from ProviderCore.base import FileProvider, Provider, SearchProvider, SearchResult +from ProviderCore.base import Provider, SearchResult _EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH") +def _class_supports_method( + plugin_class: Type[Provider], + method_name: str, + base_method: Any, +) -> bool: + try: + method = getattr(plugin_class, method_name, None) + except Exception: + return False + return callable(method) and method is not base_method + + def _repo_root() -> Path: try: return Path(__file__).resolve().parents[1] @@ -102,34 +114,34 @@ def _iter_external_plugin_entries(plugin_dir: Path) -> Iterable[Tuple[str, Path, return tuple(out) @dataclass(frozen=True) -class ProviderInfo: +class PluginInfo: """Metadata about a single plugin entry.""" canonical_name: str - provider_class: Type[Provider] + plugin_class: Type[Provider] module: str alias_names: Tuple[str, ...] = field(default_factory=tuple) @property def supports_search(self) -> bool: - return self.provider_class.search is not Provider.search + return _class_supports_method(self.plugin_class, "search", Provider.search) @property def supports_upload(self) -> bool: try: - exposed = bool(getattr(self.provider_class, "EXPOSE_AS_FILE_PROVIDER", True)) + exposed = bool(getattr(self.plugin_class, "EXPOSE_AS_FILE_PROVIDER", True)) except Exception: exposed = True - return exposed and (self.provider_class.upload is not Provider.upload) + return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload) -class ProviderRegistry: +class PluginRegistry: """Handles discovery, registration, and lookup of built-in and external plugins.""" def __init__(self, package_name: str) -> None: self.package_name = (package_name or "").strip() - self._infos: Dict[str, ProviderInfo] = {} - self._lookup: Dict[str, ProviderInfo] = {} + self._infos: Dict[str, PluginInfo] = {} + self._lookup: Dict[str, PluginInfo] = {} self._modules: set[str] = set() self._external_modules: set[str] = set() self._builtin_package_dirs: Tuple[Path, ...] = () @@ -174,7 +186,7 @@ class ProviderRegistry: return str(value or "").strip().lower() def _candidate_names(self, - provider_class: Type[Provider], + plugin_class: Type[Provider], override_name: Optional[str]) -> List[str]: names: List[str] = [] seen: set[str] = set() @@ -190,25 +202,25 @@ class ProviderRegistry: if override_name: _add(override_name) else: - _add(getattr(provider_class, "PLUGIN_NAME", None)) - _add(getattr(provider_class, "__name__", None)) + _add(getattr(plugin_class, "PLUGIN_NAME", None)) + _add(getattr(plugin_class, "__name__", None)) - for alias in getattr(provider_class, "PLUGIN_ALIASES", ()) or (): + for alias in getattr(plugin_class, "PLUGIN_ALIASES", ()) or (): _add(alias) return names def register( self, - provider_class: Type[Provider], + plugin_class: Type[Provider], *, override_name: Optional[str] = None, extra_aliases: Optional[Sequence[str]] = None, module_name: Optional[str] = None, replace: bool = False, - ) -> ProviderInfo: + ) -> PluginInfo: """Register a plugin class with canonical and alias names.""" - candidates = self._candidate_names(provider_class, override_name) + candidates = self._candidate_names(plugin_class, override_name) if not candidates: raise ValueError("plugin name candidates are required") @@ -233,10 +245,10 @@ class ProviderRegistry: alias_seen.add(normalized) alias_names.append(normalized) - info = ProviderInfo( + info = PluginInfo( canonical_name=canonical, - provider_class=provider_class, - module=module_name or getattr(provider_class, "__module__", "") or "", + plugin_class=plugin_class, + module=module_name or getattr(plugin_class, "__module__", "") or "", alias_names=tuple(alias_names), ) @@ -261,7 +273,7 @@ class ProviderRegistry: continue if not issubclass(candidate, Provider): continue - if candidate in {Provider, SearchProvider, FileProvider}: + if candidate is Provider: continue if getattr(candidate, "__module__", "") != module_name: continue @@ -311,7 +323,7 @@ class ProviderRegistry: log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr) def discover(self) -> None: - """Import and register providers from the package.""" + """Import and register plugins from the package.""" if self._discovered or not self.package_name: return @@ -376,7 +388,7 @@ class ProviderRegistry: self._sync_subclasses() return - def get(self, name: str) -> Optional[ProviderInfo]: + def get(self, name: str) -> Optional[PluginInfo]: if not name: return None @@ -398,7 +410,7 @@ class ProviderRegistry: self.discover() return self._lookup.get(normalized) - def iter_providers(self) -> Iterable[ProviderInfo]: + def iter_plugins(self) -> Iterable[PluginInfo]: self.discover() return tuple(self._infos.values()) @@ -406,12 +418,9 @@ class ProviderRegistry: return self.get(name) is not None def _sync_subclasses(self) -> None: - """Walk all Provider subclasses in memory and register them.""" + """Walk all plugin subclasses in memory and register them.""" def _walk(cls: Type[Provider]) -> None: for sub in cls.__subclasses__(): - if sub in {SearchProvider, FileProvider}: - _walk(sub) - continue try: self.register(sub) except Exception: @@ -419,16 +428,14 @@ class ProviderRegistry: _walk(sub) _walk(Provider) -REGISTRY = ProviderRegistry("plugins") +REGISTRY = PluginRegistry("plugins") PLUGIN_REGISTRY = REGISTRY -PluginInfo = ProviderInfo -PluginRegistry = ProviderRegistry @lru_cache(maxsize=512) -def _provider_url_patterns(provider_class: Type[Provider]) -> Sequence[str]: +def _plugin_url_patterns(plugin_class: Type[Provider]) -> Sequence[str]: try: - return list(provider_class.url_patterns()) + return list(plugin_class.url_patterns()) except Exception: return [] @@ -440,7 +447,7 @@ def register_plugin( aliases: Optional[Sequence[str]] = None, module_name: Optional[str] = None, replace: bool = False, -) -> ProviderInfo: +) -> PluginInfo: return REGISTRY.register( plugin_class, override_name=name, @@ -454,7 +461,7 @@ def get_plugin_class(name: str) -> Optional[Type[Provider]]: info = REGISTRY.get(name) if info is None: return None - return info.provider_class + return info.plugin_class def selection_auto_stage_for_table( @@ -481,7 +488,7 @@ def is_known_plugin_name(name: str) -> bool: def _supports_search(provider: Provider) -> bool: - return provider.__class__.search is not Provider.search + return _class_supports_method(provider.__class__, "search", Provider.search) def _supports_upload(provider: Provider) -> bool: @@ -489,7 +496,25 @@ def _supports_upload(provider: Provider) -> bool: exposed = bool(getattr(provider.__class__, "EXPOSE_AS_FILE_PROVIDER", True)) except Exception: exposed = True - return exposed and (provider.__class__.upload is not Provider.upload) + return exposed and _class_supports_method(provider.__class__, "upload", Provider.upload) + + +def _supports_capability(provider: Provider, capability: str) -> bool: + capability_key = str(capability or "").strip().lower() + if capability_key == "search": + return _supports_search(provider) + if capability_key in {"upload", "file", "file-provider"}: + return _supports_upload(provider) + return False + + +def _info_supports_capability(info: PluginInfo, capability: str) -> bool: + capability_key = str(capability or "").strip().lower() + if capability_key == "search": + return bool(info.supports_search) + if capability_key in {"upload", "file", "file-provider"}: + return bool(info.supports_upload) + return False def _normalize_choice_entry(entry: Any) -> Optional[Dict[str, Any]]: @@ -555,7 +580,7 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P return None try: - plugin = info.provider_class(config) + plugin = info.plugin_class(config) if not plugin.validate(): debug(f"[plugin] Plugin '{name}' is not available") return None @@ -567,78 +592,50 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: availability: Dict[str, bool] = {} - for info in REGISTRY.iter_providers(): + for info in REGISTRY.iter_plugins(): try: - plugin = info.provider_class(config) + plugin = info.plugin_class(config) availability[info.canonical_name] = plugin.validate() except Exception: availability[info.canonical_name] = False return availability -def get_search_plugin(name: str, - config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]: +def get_plugin_with_capability( + name: str, + capability: str, + config: Optional[Dict[str, Any]] = None, +) -> Optional[Provider]: plugin = get_plugin(name, config) if plugin is None: return None - if not _supports_search(plugin): - debug(f"[plugin] Plugin '{name}' does not support search") + if not _supports_capability(plugin, capability): + debug(f"[plugin] Plugin '{name}' does not support capability '{capability}'") return None - return plugin # type: ignore[return-value] + return plugin -def list_search_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: +def list_plugins_with_capability( + capability: str, + config: Optional[Dict[str, Any]] = None, +) -> Dict[str, bool]: availability: Dict[str, bool] = {} - for info in REGISTRY.iter_providers(): + for info in REGISTRY.iter_plugins(): try: - plugin = info.provider_class(config) + plugin = info.plugin_class(config) availability[info.canonical_name] = bool( - plugin.validate() and info.supports_search + plugin.validate() and _supports_capability(plugin, capability) ) except Exception: availability[info.canonical_name] = False return availability -def list_search_plugin_names() -> List[str]: - """Return registered search-provider names without instantiating plugins.""" +def list_plugin_names_with_capability(capability: str) -> List[str]: return sorted( info.canonical_name - for info in REGISTRY.iter_providers() - if info.supports_search - ) - - -def get_upload_plugin(name: str, - config: Optional[Dict[str, Any]] = None) -> Optional[FileProvider]: - plugin = get_plugin(name, config) - if plugin is None: - return None - if not _supports_upload(plugin): - debug(f"[plugin] Plugin '{name}' does not support upload") - return None - return plugin # type: ignore[return-value] - - -def list_upload_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: - availability: Dict[str, bool] = {} - for info in REGISTRY.iter_providers(): - try: - plugin = info.provider_class(config) - availability[info.canonical_name] = bool( - plugin.validate() and info.supports_upload - ) - except Exception: - availability[info.canonical_name] = False - return availability - - -def list_upload_plugin_names() -> List[str]: - """Return registered upload-provider names without instantiating plugins.""" - return sorted( - info.canonical_name - for info in REGISTRY.iter_providers() - if info.supports_upload + for info in REGISTRY.iter_plugins() + if _info_supports_capability(info, capability) ) @@ -677,8 +674,8 @@ def match_plugin_name_for_url(url: str) -> Optional[str]: return "openlibrary" if REGISTRY.has_name("openlibrary") else None return "internetarchive" if REGISTRY.has_name("internetarchive") else None - for info in REGISTRY.iter_providers(): - domains = _provider_url_patterns(info.provider_class) + for info in REGISTRY.iter_plugins(): + domains = _plugin_url_patterns(info.plugin_class) if not domains: continue for domain in domains: @@ -721,12 +718,10 @@ def plugin_inline_query_choices( mapping: Dict[str, List[Dict[str, Any]]] = {} info = REGISTRY.get(pname) if info is not None: - mapping = _collect_inline_choice_mapping(info.provider_class) + mapping = _collect_inline_choice_mapping(info.plugin_class) if not mapping: - plugin = get_search_plugin(pname, config) - if plugin is None: - plugin = get_plugin(pname, config) + plugin = get_plugin(pname, config) if plugin is None: return [] mapping = _collect_inline_choice_mapping(plugin) @@ -770,9 +765,9 @@ def get_plugin_for_url(url: str, def list_selection_url_prefixes() -> List[str]: prefixes: List[str] = [] seen: set[str] = set() - for info in REGISTRY.iter_providers(): + for info in REGISTRY.iter_plugins(): try: - values = info.provider_class.selection_url_prefixes() + values = info.plugin_class.selection_url_prefixes() except Exception: values = () for value in values or (): @@ -842,21 +837,17 @@ def resolve_inline_filters( __all__ = [ - "ProviderInfo", "PluginInfo", "Provider", - "SearchProvider", - "FileProvider", "SearchResult", "PluginRegistry", "PLUGIN_REGISTRY", "register_plugin", "get_plugin", "list_plugins", - "get_search_plugin", - "list_search_plugins", - "get_upload_plugin", - "list_upload_plugins", + "get_plugin_with_capability", + "list_plugins_with_capability", + "list_plugin_names_with_capability", "match_plugin_name_for_url", "get_plugin_for_url", "list_selection_url_prefixes", diff --git a/SYS/detail_view_helpers.py b/SYS/detail_view_helpers.py index beb979c..5480c64 100644 --- a/SYS/detail_view_helpers.py +++ b/SYS/detail_view_helpers.py @@ -3,8 +3,41 @@ from __future__ import annotations from typing import Any, Iterable, Optional, Sequence +_ACRONYM_LABELS = { + "id": "ID", + "ids": "IDs", + "url": "URL", + "urls": "URLs", + "api": "API", + "http": "HTTP", + "https": "HTTPS", + "ftp": "FTP", + "ftps": "FTPS", + "scp": "SCP", + "ssh": "SSH", + "ip": "IP", + "mpv": "MPV", +} + + def _labelize_key(key: str) -> str: - return str(key or "").replace("_", " ").title() + parts = [part for part in str(key or "").replace("_", " ").split() if part] + normalized: list[str] = [] + for part in parts: + lowered = part.lower() + normalized.append(_ACRONYM_LABELS.get(lowered, part.title())) + return " ".join(normalized) + + +def _has_display_value(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + text = value.strip() + return bool(text and text.lower() not in {"", "null", "none"}) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return any(_has_display_value(item) for item in value) + return True def _normalize_tags_value(tags: Any) -> Optional[str]: @@ -45,7 +78,7 @@ def prepare_detail_metadata( if str(key).startswith("_") or key in {"selection_action", "selection_args"}: continue label = _labelize_key(str(key)) - if label not in metadata and value is not None: + if label not in metadata and _has_display_value(value): metadata[label] = value if title: @@ -62,7 +95,7 @@ def prepare_detail_metadata( metadata["Tags"] = tags_text for key, value in (extra_fields or {}).items(): - if value is not None: + if _has_display_value(value): metadata[str(key)] = value return metadata @@ -77,6 +110,7 @@ def create_detail_view( init_command: Optional[tuple[str, Sequence[str]]] = None, max_columns: Optional[int] = None, exclude_tags: bool = False, + detail_order: Optional[Sequence[str]] = None, value_case: Optional[str] = "preserve", perseverance: bool = True, ) -> Any: @@ -87,6 +121,8 @@ def create_detail_view( kwargs["max_columns"] = max_columns if exclude_tags: kwargs["exclude_tags"] = True + if detail_order is not None: + kwargs["detail_order"] = list(detail_order) table = ItemDetailView(title, **kwargs) if table_name: @@ -101,4 +137,46 @@ def create_detail_view( if init_command: name, args = init_command table = table.init_command(name, list(args)) - return table \ No newline at end of file + return table + + +def render_selection_detail_view( + *, + ctx: Any, + item: Any, + title: str, + metadata: dict[str, Any], + table_name: Optional[str] = None, + detail_order: Optional[Sequence[str]] = None, + value_case: Optional[str] = "preserve", + exclude_tags: bool = False, +) -> bool: + from SYS.rich_display import stdout_console + + detail_view = create_detail_view( + title, + metadata, + table_name=table_name, + detail_order=detail_order, + value_case=value_case, + exclude_tags=exclude_tags, + ) + + payload = item.to_dict() if hasattr(item, "to_dict") else item + try: + if hasattr(ctx, "set_last_result_items_only") and isinstance(payload, dict): + ctx.set_last_result_items_only([payload]) + except Exception: + pass + + try: + try: + detail_view.title = "" + detail_view.header_lines = [] + except Exception: + pass + stdout_console().print() + stdout_console().print(detail_view.to_rich()) + except Exception: + return False + return True \ No newline at end of file diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 76478b3..7dff2cb 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1478,13 +1478,18 @@ class PipelineExecutor: config: Any, selected_items: list, *, - stage_is_last: bool + stage_is_last: bool, + source_command: Any = None, + prefer_detail_fallback: bool = False, ) -> bool: if not stage_is_last: return False candidates: list[str] = [] seen: set[str] = set() + current_table = None + table_meta = None + table_type = "" def _add(value) -> None: try: @@ -1504,6 +1509,8 @@ class PipelineExecutor: table if current_table and hasattr(current_table, "table") else None ) + if current_table and hasattr(current_table, "table"): + table_type = str(getattr(current_table, "table", "") or "").strip() # Prefer an explicit plugin hint from table metadata when available. # This keeps @N selectors working even when row payloads don't carry a @@ -1516,6 +1523,7 @@ class PipelineExecutor: ) except Exception: meta = None + table_meta = meta if isinstance(meta, dict) else None if isinstance(meta, dict): _add(meta.get("plugin")) _add(meta.get("provider")) @@ -1585,6 +1593,26 @@ class PipelineExecutor: if handled: return True + if prefer_detail_fallback: + detail_renderer = getattr(provider, "show_selection_details", None) + if callable(detail_renderer): + try: + detail_handled = bool( + detail_renderer( + selected_items, + ctx=ctx, + stage_is_last=True, + source_command=str(source_command or ""), + table_type=table_type, + table_metadata=table_meta, + ) + ) + except Exception as exc: + logger.exception("%s detail fallback failed during selection: %s", key, exc) + return True + if detail_handled: + return True + @staticmethod def _maybe_expand_plugin_selection( selected_items: List[Any], @@ -2180,10 +2208,12 @@ class PipelineExecutor: filtered = expanded if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(not stages)): + ctx, + config, + filtered, + stage_is_last=(not stages), + source_command=source_cmd, + prefer_detail_fallback=bool(prefer_row_action and not stages and len(selection_indices) == 1)): return False, None from SYS.pipe_object import coerce_to_pipe_object @@ -2204,7 +2234,7 @@ class PipelineExecutor: except Exception: logger.exception("Failed to record Applied @N selection log step (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) - # Auto-insert downloader stages for provider tables. + # Auto-insert downloader stages for plugin tables. try: current_table = ctx.get_current_stage_table() if current_table is None and hasattr(ctx, "get_display_table"): @@ -2360,7 +2390,7 @@ class PipelineExecutor: # Multi-selection fallback: if any selected row declares a # download-file action, insert a generic download-file stage. - # This keeps provider-specific behavior in provider metadata. + # This keeps plugin-specific behavior in plugin metadata. if (not inserted_provider_download) and len(selection_indices) > 1: try: has_download_row_action = False diff --git a/SYS/plugin_config.py b/SYS/plugin_config.py index e5967ef..4095f5d 100644 --- a/SYS/plugin_config.py +++ b/SYS/plugin_config.py @@ -68,10 +68,6 @@ def get_plugin_schema(plugin_name: str) -> List[ConfigField]: return _call_schema(plugin_class, f"plugin '{plugin_name}'") -def get_provider_schema(provider_name: str) -> List[ConfigField]: - return get_plugin_schema(provider_name) - - def get_tool_schema(tool_name: str) -> List[ConfigField]: tool_name = str(tool_name or "").strip() if not tool_name: @@ -149,10 +145,6 @@ def build_default_plugin_config(plugin_name: str) -> Dict[str, Any]: return config -def build_default_provider_config(provider_name: str) -> Dict[str, Any]: - return build_default_plugin_config(provider_name) - - def build_default_tool_config(tool_name: str) -> Dict[str, Any]: config: Dict[str, Any] = {} for field in get_tool_schema(tool_name): @@ -215,10 +207,6 @@ def get_configurable_plugin_types() -> List[str]: return sorted(set(options)) -def get_configurable_provider_types() -> List[str]: - return get_configurable_plugin_types() - - def get_configurable_tool_types() -> List[str]: options: List[str] = [] try: diff --git a/SYS/result_table.py b/SYS/result_table.py index 29631a8..41f718d 100644 --- a/SYS/result_table.py +++ b/SYS/result_table.py @@ -724,7 +724,7 @@ class Table: """Table type (e.g., 'youtube', 'soulseek') for context-aware selection logic.""" self.table_metadata: Dict[str, Any] = {} - """Optional provider/table metadata (e.g., provider name, view).""" + """Optional plugin/table metadata (e.g., plugin name, view).""" self.value_case: str = "preserve" """Display-only value casing: 'lower', 'upper', or 'preserve' (default).""" @@ -754,12 +754,12 @@ class Table: return self def set_table_metadata(self, metadata: Optional[Dict[str, Any]]) -> "Table": - """Attach provider/table metadata for downstream selection logic.""" + """Attach plugin/table metadata for downstream selection logic.""" self.table_metadata = dict(metadata or {}) return self def get_table_metadata(self) -> Dict[str, Any]: - """Return attached provider/table metadata (copy to avoid mutation).""" + """Return attached plugin/table metadata (copy to avoid mutation).""" try: return dict(self.table_metadata) except Exception: @@ -2223,6 +2223,34 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]: return {} out = {} + + def _merge_columns(columns_value: Any) -> None: + if not isinstance(columns_value, (list, tuple)): + return + for column in columns_value: + label = None + value = None + if isinstance(column, (list, tuple)) and len(column) >= 2: + label, value = column[0], column[1] + elif isinstance(column, dict): + label = column.get("name") or column.get("label") or column.get("key") + value = column.get("value") + else: + label = getattr(column, "name", None) + value = getattr(column, "value", None) + + label_text = str(label or "").strip() + if not label_text or value is None: + continue + + value_text = str(value).strip() + if not value_text: + continue + + normalized = label_text.lower() + if any(str(existing or "").strip().lower() == normalized for existing in out): + continue + out[label_text] = value_text # Handle ResultModel specifically for better detail display if ResultModel is not None and isinstance(item, ResultModel): @@ -2231,6 +2259,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]: if item.ext: out["Ext"] = item.ext if item.size_bytes: out["Size"] = format_mb(item.size_bytes) if item.source: out["Store"] = item.source + _merge_columns(getattr(item, "columns", None)) # Merge internal metadata dict if item.metadata: @@ -2256,6 +2285,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]: # Fallback to existing extraction logic for legacy objects/dicts # Convert once and reuse throughout to avoid repeated _as_dict() calls data = _as_dict(item) or {} + _merge_columns(data.get("columns")) # Use existing extractors from match-standard result table columns title = extract_title_value(item) @@ -2350,12 +2380,14 @@ class ItemDetailView(Table): item_metadata: Optional[Dict[str, Any]] = None, detail_title: Optional[str] = None, exclude_tags: bool = False, + detail_order: Optional[List[str]] = None, **kwargs ): super().__init__(title, **kwargs) self.item_metadata = item_metadata or {} self.detail_title = detail_title self.exclude_tags = exclude_tags + self.detail_order = [str(value) for value in (detail_order or []) if str(value or "").strip()] def to_rich(self): """Render the item details panel above the standard results table.""" @@ -2406,8 +2438,26 @@ class ItemDetailView(Table): return Group(*renderables) - # Canonical display order for metadata - order = ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"] + def _has_renderable_value(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + text = value.strip() + return bool(text and text.lower() not in {"", "null", "none"}) + if isinstance(value, (list, tuple, set)): + return any(_has_renderable_value(item) for item in value) + return True + + # Canonical display order for metadata; plugin-specific detail views can + # prepend a preferred order without needing to reimplement rendering. + order: List[str] = [] + seen_order: set[str] = set() + for key in list(self.detail_order or []) + ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"]: + normalized = str(key or "").strip().lower() + if not normalized or normalized in seen_order: + continue + seen_order.add(normalized) + order.append(str(key)) has_details = False # Add ordered items first @@ -2431,19 +2481,16 @@ class ItemDetailView(Table): else: val = "\n".join([f"[dim]→[/dim] {r}" for r in val]) - if val is not None and val != "": + if _has_renderable_value(val): details_table.add_row(f"{key}:", str(val)) has_details = True - elif key in ["Url", "Relations", "Ext"]: - # Show for these important identifier fields if blank - details_table.add_row(f"{key}:", "[dim][/dim]") - has_details = True # Add any remaining metadata not in the canonical list + ordered_keys = {x.lower() for x in order} for k, v in self.item_metadata.items(): k_norm = k.lower() - if k_norm not in [x.lower() for x in order] and v and k_norm not in ["tags", "tag"]: - label = k.capitalize() if len(k) > 1 else k.upper() + if k_norm not in ordered_keys and _has_renderable_value(v) and k_norm not in ["tags", "tag"]: + label = str(k or "") details_table.add_row(f"{label}:", str(v)) has_details = True diff --git a/SYS/rich_display.py b/SYS/rich_display.py index f7be249..c3b20f8 100644 --- a/SYS/rich_display.py +++ b/SYS/rich_display.py @@ -77,26 +77,26 @@ def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]: _STDERR_CONSOLE = previous_stderr -def show_provider_config_panel( - provider_names: str | List[str], +def show_plugin_config_panel( + plugin_names: str | List[str], ) -> None: - """Show a Rich panel explaining how to configure providers.""" + """Show a Rich panel explaining how to configure plugins.""" from rich.table import Table as RichTable from rich.console import Group - if isinstance(provider_names, str): - providers = [p.strip() for p in provider_names.split(",")] + if isinstance(plugin_names, str): + plugins = [p.strip() for p in plugin_names.split(",")] else: - providers = provider_names + plugins = plugin_names table = RichTable.grid(padding=(0, 1)) table.add_column(style="bold red") - for provider in providers: - table.add_row(f" • {provider}") + for plugin in plugins: + table.add_row(f" • {plugin}") group = Group( - Text("The following providers are not configured and cannot be used:\n"), + Text("The following plugins are not configured and cannot be used:\n"), table, Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.") ) @@ -147,17 +147,17 @@ def show_store_config_panel( stdout_console().print(panel) -def show_available_providers_panel(provider_names: List[str]) -> None: +def show_available_plugins_panel(plugin_names: List[str]) -> None: """Show a Rich panel listing available/configured plugins.""" from rich.columns import Columns from rich.console import Group - if not provider_names: + if not plugin_names: return # Use Columns to display them efficiently in the panel cols = Columns( - [f"[bold green] \u2713 [/bold green]{p}" for p in sorted(provider_names)], + [f"[bold green] \u2713 [/bold green]{p}" for p in sorted(plugin_names)], equal=True, column_first=True, expand=True diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index 06aabec..ecc7fc7 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -4,6 +4,7 @@ import re import sys import tempfile import shutil +from collections.abc import Mapping, Sequence as SequenceABC from collections import deque from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple @@ -169,7 +170,7 @@ class HydrusNetwork(Store): api_key: Hydrus Client API access key url: Hydrus client URL (e.g., 'http://192.168.1.230:45869') """ - from API.HydrusNetwork import HydrusNetwork as HydrusClient + from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient if instance_name is None and NAME is not None: instance_name = str(NAME) @@ -713,7 +714,7 @@ class HydrusNetwork(Store): """Best-effort URL search by scanning Hydrus metadata with include_file_url=True.""" try: - from API.HydrusNetwork import _generate_hydrus_url_variants + from plugins.hydrusnetwork.api import _generate_hydrus_url_variants except Exception: _generate_hydrus_url_variants = None # type: ignore[assignment] @@ -808,7 +809,7 @@ class HydrusNetwork(Store): self._has_url_predicate = True except Exception as exc: try: - from API.HydrusNetwork import HydrusRequestError + from plugins.hydrusnetwork.api import HydrusRequestError if isinstance(exc, HydrusRequestError) and getattr(exc, "status", None) == 400: self._has_url_predicate = False @@ -1844,7 +1845,7 @@ class HydrusNetwork(Store): extracted_tags = self._extract_tags_from_hydrus_meta( meta, service_key=None, - service_name="my tags", + service_name=None, ) for raw_tag in extracted_tags: tag_text = str(raw_tag or "").strip() @@ -2201,7 +2202,7 @@ class HydrusNetwork(Store): return [] try: - from API.HydrusNetwork import HydrusRequestSpec, _generate_hydrus_url_variants + from plugins.hydrusnetwork.api import HydrusRequestSpec, _generate_hydrus_url_variants except Exception: return [] @@ -2339,7 +2340,7 @@ class HydrusNetwork(Store): try: return client.get_url_info(u) # type: ignore[attr-defined] except Exception: - from API.HydrusNetwork import HydrusRequestSpec + from plugins.hydrusnetwork.api import HydrusRequestSpec spec = HydrusRequestSpec( method="GET", @@ -2563,7 +2564,7 @@ class HydrusNetwork(Store): meta: Dict[str, Any], service_key: Optional[str], - service_name: str + service_name: Optional[str] ) -> List[str]: """Extract current tags from Hydrus metadata dict. @@ -2571,7 +2572,7 @@ class HydrusNetwork(Store): Falls back to storage_tags status '0' (current). """ tags_payload = meta.get("tags") - if not isinstance(tags_payload, dict): + if not isinstance(tags_payload, Mapping): return [] desired_service_name = str(service_name or "").strip().lower() @@ -2593,20 +2594,20 @@ class HydrusNetwork(Store): out.append(cleaned) def _collect_current(container: Any, out: List[str]) -> None: - if isinstance(container, list): + if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)): for tag in container: _append_tag(out, tag) return - if isinstance(container, dict): + if isinstance(container, Mapping): current = container.get("0") if current is None: current = container.get(0) - if isinstance(current, list): + if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)): for tag in current: _append_tag(out, tag) def _collect_service_data(service_data: Any, out: List[str]) -> None: - if not isinstance(service_data, dict): + if not isinstance(service_data, Mapping): return display = ( @@ -2630,7 +2631,7 @@ class HydrusNetwork(Store): if not collected and desired_service_name: for maybe_service in tags_payload.values(): - if not isinstance(maybe_service, dict): + if not isinstance(maybe_service, Mapping): continue svc_name = str( maybe_service.get("service_name") @@ -2642,11 +2643,11 @@ class HydrusNetwork(Store): names_map = tags_payload.get("service_keys_to_names") statuses_map = tags_payload.get("service_keys_to_statuses_to_tags") - if isinstance(statuses_map, dict): + if isinstance(statuses_map, Mapping): keys_to_collect: List[str] = [] if desired_service_key: keys_to_collect.append(desired_service_key) - if desired_service_name and isinstance(names_map, dict): + if desired_service_name and isinstance(names_map, Mapping): for raw_key, raw_name in names_map.items(): if str(raw_name or "").strip().lower() == desired_service_name: keys_to_collect.append(str(raw_key)) @@ -2663,7 +2664,7 @@ class HydrusNetwork(Store): _collect_service_data(maybe_service, collected) top_level_tags = meta.get("tags_flat") - if isinstance(top_level_tags, list): + if isinstance(top_level_tags, SequenceABC) and not isinstance(top_level_tags, (str, bytes, bytearray, Mapping)): _collect_current(top_level_tags, collected) deduped: List[str] = [] @@ -2682,7 +2683,7 @@ class HydrusNetwork(Store): tags = HydrusNetwork._extract_tags_from_hydrus_meta( meta, service_key=None, - service_name="my tags", + service_name=None, ) normalized_tags: List[str] = [] diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index 8512a58..f6d0251 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -22,10 +22,10 @@ from SYS.config import ( from SYS.database import db from SYS.logger import log, debug from SYS.plugin_config import ( - build_default_provider_config, + build_default_plugin_config, build_default_store_config, build_default_tool_config, - get_configurable_provider_types, + get_configurable_plugin_types, get_configurable_store_types, get_configurable_tool_types, get_global_schema, @@ -161,7 +161,7 @@ class ConfigModal(ModalScreen): # Load config from the workspace root (parent of SYS) self.config_data = load_config() self.current_category = "globals" - self.editing_item_type = None # 'store' or 'provider' + self.editing_item_type = None # 'store' or 'plugin' self.editing_item_name = None self._button_id_map = {} self._provider_button_map: Dict[str, tuple[str, str]] = {} @@ -598,14 +598,14 @@ class ConfigModal(ModalScreen): row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) idx += 1 - if item_type == "provider" and isinstance(item_name, str): - provider = self._instantiate_provider_for_editor(item_name, self.config_data) + if item_type == "plugin" and isinstance(item_name, str): + provider = self._instantiate_plugin_for_editor(item_name, self.config_data) if provider is not None: provider_actions = provider.config_actions() or [] if provider_actions: container.mount(Rule()) container.mount(Label(f"{provider.label} helpers", classes="config-label")) - helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip() + helper_text = str(provider.config_helper_text() or "Use these helpers to validate plugin settings.").strip() status = Static(helper_text, id="provider-status") container.mount(status) self._provider_status = status @@ -626,7 +626,7 @@ class ConfigModal(ModalScreen): ) if ( - item_type == "provider" + item_type == "plugin" and isinstance(item_name, str) and item_name.strip().lower() == "matrix" ): @@ -870,12 +870,12 @@ class ConfigModal(ModalScreen): self.refresh_view() elif bid in self._provider_button_map: provider_name, action_id = self._provider_button_map[bid] - self._request_provider_action(provider_name, action_id) + self._request_plugin_action(provider_name, action_id) elif bid == "add-store-btn": options = get_configurable_store_types() self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected) elif bid == "add-provider-btn": - options = get_configurable_provider_types() + options = get_configurable_plugin_types() self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected) elif bid == "add-tool-btn": options = get_configurable_tool_types() or ["ytdlp"] @@ -971,46 +971,46 @@ class ConfigModal(ModalScreen): else: self.notify("Clipboard not supported in this terminal", severity="warning") - def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]: + def _instantiate_plugin_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]: try: - provider_class = get_plugin_class(provider_name) + plugin_class = get_plugin_class(provider_name) except Exception: - provider_class = None - if provider_class is None: + plugin_class = None + if plugin_class is None: return None try: - return provider_class(config_data or self.config_data) + return plugin_class(config_data or self.config_data) except Exception: - logger.exception("Failed to instantiate provider '%s' for config helper", provider_name) + logger.exception("Failed to instantiate plugin '%s' for config helper", provider_name) return None - def _request_provider_action(self, provider_name: str, action_id: str) -> None: + def _request_plugin_action(self, provider_name: str, action_id: str) -> None: if self._provider_action_running: return self._synchronize_inputs_to_config() self._provider_action_running = True if self._provider_status is not None: self._provider_status.update(f"Running {action_id.replace('_', ' ')}…") - self._provider_action_background(provider_name, action_id, deepcopy(self.config_data)) + self._plugin_action_background(provider_name, action_id, deepcopy(self.config_data)) @work(thread=True) - def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None: + def _plugin_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None: try: - provider = self._instantiate_provider_for_editor(provider_name, config_snapshot) + provider = self._instantiate_plugin_for_editor(provider_name, config_snapshot) if provider is None: - raise RuntimeError(f"Provider '{provider_name}' is unavailable") + raise RuntimeError(f"Plugin '{provider_name}' is unavailable") result = provider.run_config_action(action_id) if not isinstance(result, dict): - result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."} + result = {"ok": False, "message": f"Plugin '{provider_name}' returned an invalid config action result."} except Exception as exc: - result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."} + result = {"ok": False, "message": str(exc) or f"Plugin action '{action_id}' failed."} try: - self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result) + self.app.call_from_thread(self._plugin_action_complete, provider_name, action_id, result) except Exception: - self._provider_action_complete(provider_name, action_id, result) + self._plugin_action_complete(provider_name, action_id, result) - def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None: + def _plugin_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None: self._provider_action_running = False ok = bool(result.get("ok")) message = str(result.get("message") or f"Provider action '{action_id}' finished.") @@ -1075,11 +1075,11 @@ class ConfigModal(ModalScreen): if "provider" not in self.config_data: self.config_data["provider"] = {} - # For providers, they are usually top-level entries in 'provider' dict + # Plugins are configured under the top-level 'provider' dict for now. if ptype not in self.config_data["provider"]: - self.config_data["provider"][ptype] = build_default_provider_config(ptype) + self.config_data["provider"][ptype] = build_default_plugin_config(ptype) - self.editing_item_type = "provider" + self.editing_item_type = "plugin" self.editing_item_name = ptype self.refresh_view() diff --git a/TUI/modalscreen/search.py b/TUI/modalscreen/search.py index ccd9c1e..ee39739 100644 --- a/TUI/modalscreen/search.py +++ b/TUI/modalscreen/search.py @@ -16,7 +16,7 @@ import asyncio sys.path.insert(0, str(Path(__file__).parent.parent)) from SYS.config import load_config, resolve_output_dir from SYS.result_table import Table -from ProviderCore.registry import get_search_plugin +from ProviderCore.registry import get_plugin_with_capability logger = logging.getLogger(__name__) @@ -174,7 +174,7 @@ class SearchModal(ModalScreen): self.current_worker.log_step(f"Connecting to {source}...") try: - provider = get_search_plugin(source) + provider = get_plugin_with_capability(source, "search") if not provider: logger.error(f"[search-modal] Provider not available: {source}") if self.current_worker: @@ -380,7 +380,7 @@ class SearchModal(ModalScreen): config = load_config() output_dir = resolve_output_dir(config) - provider = get_search_plugin("openlibrary", config=config) + provider = get_plugin_with_capability("openlibrary", "search", config=config) if not provider: logger.error("[search-modal] Provider not available: openlibrary") return diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index d69e518..b7d7f22 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -190,10 +190,18 @@ class SharedArgs: name="store", type="enum", choices=[], # Dynamically populated via get_store_choices() - description="Selects store", + description="Selects a storage backend", query_key="store", ) + INSTANCE = CmdletArg( + name="instance", + type="string", + description="Selects a plugin instance", + query_key="instance", + query_aliases=["store"], + ) + URL = CmdletArg( name="url", type="string", @@ -1410,7 +1418,7 @@ def fetch_hydrus_metadata( Eliminates repeated boilerplate: client initialization, error handling, metadata extraction. Args: - config: Configuration object used to resolve the Hydrus provider/store + config: Configuration object used to resolve the Hydrus plugin/store hash_hex: File hash to fetch metadata for store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client. hydrus_client: Optional explicit Hydrus client. When provided, takes precedence. diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 5c0ce1a..c0619be 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -15,6 +15,7 @@ from SYS.logger import log, debug, debug_panel, is_debug_enabled from SYS.payload_builders import build_table_result_payload from SYS.pipeline_progress import PipelineProgress from SYS.result_publication import overlay_existing_result_table, publish_result_table +from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS from Store import Store from API.HTTP import _download_direct_file @@ -178,10 +179,11 @@ class Add_File(Cmdlet): summary= "Ingest a local media file to a store backend, upload plugin, or local directory.", usage= - "add-file (-path | ) (-storage | -plugin ) [-delete]", + "add-file (-path | ) (-store | -plugin ) [-instance NAME] [-delete]", arg=[ SharedArgs.PATH, SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.URL, SharedArgs.PLUGIN, CmdletArg( @@ -194,7 +196,7 @@ class Add_File(Cmdlet): ], detail=[ "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", - "- Storage location options (use -storage):", + "- Storage location options (use -store):", " hydrus: Upload to Hydrus database with metadata tagging", " local: Copy file to local directory", " : Copy file to specified directory", @@ -202,9 +204,12 @@ class Add_File(Cmdlet): " 0x0: Upload to 0x0.st for temporary hosting", " file.io: Upload to file.io for temporary hosting", " internetarchive: Upload to archive.org (optional tag: ia: to upload into an existing item)", + "- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf", + "- In plugin mode, -store is still accepted as a compatibility alias for -instance .", ], examples=[ 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial', + 'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf', ], exec=self.run, ) @@ -223,9 +228,12 @@ class Add_File(Cmdlet): path_arg = parsed.get("path") location = parsed.get("store") + plugin_instance = parsed.get("instance") source_url_arg = parsed.get("url") plugin_name = parsed.get("plugin") delete_after = parsed.get("delete", False) + if plugin_name and not plugin_instance and location: + plugin_instance = location # Convenience: when piping a file into add-file, allow `-path ` # to act as the destination export directory. @@ -412,6 +420,7 @@ class Add_File(Cmdlet): ("items", total_items), ("location", location), ("plugin", plugin_name), + ("instance", plugin_instance), ("delete", delete_after), ], border_style="cyan", @@ -647,6 +656,7 @@ class Add_File(Cmdlet): code = self._handle_plugin_upload( media_path, plugin_name, + plugin_instance, pipe_obj, config, delete_after_item @@ -1442,9 +1452,9 @@ class Add_File(Cmdlet): if not plugin_key: return None, None, None - from ProviderCore.registry import get_search_plugin + from ProviderCore.registry import get_plugin - plugin = get_search_plugin(plugin_key, config) + plugin = get_plugin(plugin_key, config) if plugin is None: return None, None, None @@ -1762,6 +1772,7 @@ class Add_File(Cmdlet): *, hash_value: str, store: str, + provider: Optional[str] = None, path: Optional[str], tag: List[str], title: Optional[str], @@ -1770,6 +1781,7 @@ class Add_File(Cmdlet): ) -> None: pipe_obj.hash = hash_value pipe_obj.store = store + pipe_obj.provider = provider pipe_obj.is_temp = False pipe_obj.path = path pipe_obj.tag = tag @@ -2180,23 +2192,42 @@ class Add_File(Cmdlet): def _handle_plugin_upload( media_path: Path, plugin_name: str, + instance_name: Optional[str], pipe_obj: models.PipeObject, config: Dict[str, Any], delete_after: bool, ) -> int: """Handle uploading via an upload plugin (e.g. 0x0).""" - from ProviderCore.registry import get_upload_plugin + from ProviderCore.registry import ( + get_plugin_with_capability, + list_plugin_names_with_capability, + list_plugins_with_capability, + ) log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr) try: - file_provider = get_upload_plugin(plugin_name, config) + file_provider = get_plugin_with_capability(plugin_name, "upload", config) if not file_provider: - log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr) + available_map = list_plugins_with_capability("upload", config) + known_upload_plugins = set(list_plugin_names_with_capability("upload")) + available_uploads = [name for name, enabled in available_map.items() if enabled and name in known_upload_plugins] + + if str(plugin_name or "").strip().lower() in known_upload_plugins: + show_plugin_config_panel([plugin_name]) + else: + log(f"Upload plugin '{plugin_name}' is not available or does not support upload", file=sys.stderr) + + if available_uploads: + show_available_plugins_panel(sorted(available_uploads)) return 1 - hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj) + hoster_url = file_provider.upload( + str(media_path), + pipe_obj=pipe_obj, + instance=instance_name, + ) log(f"File uploaded: {hoster_url}", file=sys.stderr) f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None) @@ -2209,6 +2240,7 @@ class Add_File(Cmdlet): extra_updates: Dict[str, Any] = { "plugin": plugin_name, + "instance": instance_name, "plugin_url": hoster_url, } if isinstance(pipe_obj.extra, dict): @@ -2222,7 +2254,8 @@ class Add_File(Cmdlet): Add_File._update_pipe_object_destination( pipe_obj, hash_value=f_hash or "unknown", - store=plugin_name or "plugin", + store="", + provider=plugin_name or None, path=file_path, tag=pipe_obj.tag, title=pipe_obj.title or (media_path.name if media_path else None), diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index dafe35e..c2e9a63 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -55,12 +55,13 @@ class Download_File(Cmdlet): name="download-file", summary="Download files or streaming media", usage= - "download-file [-path DIR] [options] OR @N | download-file [-path DIR|DIR] [options]", + "download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]", alias=["dl-file", "download-http"], arg=[ SharedArgs.URL, SharedArgs.PLUGIN, + SharedArgs.INSTANCE, SharedArgs.PATH, SharedArgs.QUERY, QueryArg( @@ -85,6 +86,7 @@ class Download_File(Cmdlet): ], detail=[ "Download files directly via HTTP or streaming media via yt-dlp.", + "Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.", "For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.", ], exec=self.run, @@ -522,13 +524,13 @@ class Download_File(Cmdlet): config: Dict[str, Any], ) -> List[Any]: - get_search_plugin = registry.get("get_search_plugin") + get_provider = registry.get("get_plugin") expanded_items: List[Any] = [] for item in piped_items: try: provider_key = self._provider_key_from_item(item) - provider = get_search_plugin(provider_key, config) if provider_key and get_search_plugin else None + provider = get_provider(provider_key, config) if provider_key and get_provider else None # Generic hook: If provider has expand_item(item), use it. if provider and hasattr(provider, "expand_item") and callable(provider.expand_item): @@ -566,7 +568,7 @@ class Download_File(Cmdlet): ) -> tuple[int, int]: downloaded_count = 0 queued_magnet_submissions = 0 - get_search_plugin = registry.get("get_search_plugin") + get_provider = registry.get("get_plugin") SearchResult = registry.get("SearchResult") expanded_items = self._expand_provider_items( @@ -622,15 +624,15 @@ class Download_File(Cmdlet): transfer_label = label - # If this looks like a provider item and providers are available, prefer provider.download() + # If this looks like a plugin-owned item and a plugin is available, prefer plugin.download(). downloaded_path: Optional[Path] = None attempted_provider_download = False provider_sr = None provider_obj = None provider_key = self._provider_key_from_item(item) - if provider_key and get_search_plugin and SearchResult: - # Reuse helper to derive the provider key from table/provider/source hints. - provider_obj = get_search_plugin(provider_key, config) + if provider_key and get_provider and SearchResult: + # Reuse helper to derive the plugin key from table/plugin/source hints. + provider_obj = get_provider(provider_key, config) if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): try: @@ -697,7 +699,7 @@ class Download_File(Cmdlet): ) continue - # Allow providers to add/enrich tags and metadata during download. + # Allow plugins to add or enrich tags and metadata during download. if provider_sr is not None: try: sr_md = getattr(provider_sr, "full_metadata", None) @@ -838,9 +840,9 @@ class Download_File(Cmdlet): notes: Optional[Dict[str, str]] = None try: if isinstance(full_metadata, dict): - # Providers attach pre-built notes under the generic "_notes" key + # Plugins attach pre-built notes under the generic "_notes" key # (e.g. Tidal sets {"lyric": subtitles} during download enrichment). - # This keeps provider-specific metadata handling inside the provider. + # This keeps plugin-specific metadata handling inside the plugin. _provider_notes = full_metadata.get("_notes") if isinstance(_provider_notes, dict) and _provider_notes: notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v} @@ -949,7 +951,6 @@ class Download_File(Cmdlet): return { "get_plugin": getattr(provider_registry, "get_plugin", None), - "get_search_plugin": getattr(provider_registry, "get_search_plugin", None), "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None), "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None), "SearchResult": SearchResult, @@ -957,7 +958,6 @@ class Download_File(Cmdlet): except Exception: return { "get_plugin": None, - "get_search_plugin": None, "match_plugin_name_for_url": None, "list_selection_url_prefixes": None, "SearchResult": None, diff --git a/cmdlet/get_tag.py b/cmdlet/get_tag.py index e937694..cfcca32 100644 --- a/cmdlet/get_tag.py +++ b/cmdlet/get_tag.py @@ -15,10 +15,10 @@ import sys from SYS.logger import log, debug from plugins.metadata_provider import ( - get_default_subject_scrape_provider, - get_metadata_provider, - get_metadata_provider_for_url, - list_metadata_providers, + get_default_subject_scrape_plugin, + get_metadata_plugin, + get_metadata_plugin_for_url, + list_metadata_plugins, scrape_isbn_metadata, scrape_openlibrary_metadata, ) @@ -393,33 +393,33 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: scrape_url = parsed_args.get("scrape") scrape_requested = scrape_flag_present or scrape_url is not None - # Handle URL or provider scraping mode. + # Handle URL or metadata-plugin scraping mode. if scrape_requested: import json as json_module scrape_target = str(scrape_url or "").strip() if scrape_url is not None else "" - provider = None + plugin = None if scrape_target.startswith(("http://", "https://")): - provider = get_metadata_provider_for_url(scrape_target, config) - if provider is None: - log("No metadata provider can scrape this URL", file=sys.stderr) + plugin = get_metadata_plugin_for_url(scrape_target, config) + if plugin is None: + log("No metadata plugin can scrape this URL", file=sys.stderr) return 1 - payload = provider.scrape_url_payload(scrape_target) + payload = plugin.scrape_url_payload(scrape_target) if not isinstance(payload, dict): - log(f"No metadata extracted from URL via {provider.name}", file=sys.stderr) + log(f"No metadata extracted from URL via {plugin.name}", file=sys.stderr) return 1 print(json_module.dumps(payload, ensure_ascii=False)) return 0 if scrape_target: - provider = get_metadata_provider(scrape_target, config) + plugin = get_metadata_plugin(scrape_target, config) else: - provider = get_default_subject_scrape_provider(config) - if provider is None: + plugin = get_default_subject_scrape_plugin(config) + if plugin is None: if scrape_target: - log(f"Unknown metadata provider: {scrape_target}", file=sys.stderr) + log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr) else: - log("No default metadata provider is available for subject scraping", file=sys.stderr) + log("No default metadata plugin is available for subject scraping", file=sys.stderr) return 1 backend = None @@ -548,7 +548,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: query_hint = resolved_subject_query or identifier_query or combined_query or title_hint if not query_hint: log( - f"No query could be resolved for metadata provider '{provider.name}'", + f"No query could be resolved for metadata plugin '{provider.name}'", file=sys.stderr ) return 1 @@ -749,9 +749,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: ) return 0 - provider_for_apply = get_metadata_provider(str(result_provider), config) - if provider_for_apply is not None: - apply_tags = provider_for_apply.filter_tags_for_store_apply( + plugin_for_apply = get_metadata_plugin(str(result_provider), config) + if plugin_for_apply is not None: + apply_tags = plugin_for_apply.filter_tags_for_store_apply( [str(t) for t in result_tags if t is not None] ) else: @@ -946,7 +946,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: _SCRAPE_CHOICES = [] try: - _SCRAPE_CHOICES = sorted(list_metadata_providers().keys()) + _SCRAPE_CHOICES = sorted(list_metadata_plugins().keys()) except Exception: _SCRAPE_CHOICES = [ "itunes", @@ -1000,7 +1000,7 @@ class Get_Tag(Cmdlet): ' -query: Override hash to look up in Hydrus (use: -query "hash:")', " -store: Store result to key for downstream pipeline", " -emit: Quiet mode (no interactive selection)", - " -scrape: Scrape metadata from URL or metadata provider", + " -scrape: Scrape metadata from URL or metadata plugin", ], exec=self.run, ) diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index ec52e53..8671237 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -1,4 +1,4 @@ -"""search-file cmdlet: Search for files in storage backends (Hydrus).""" +"""search-file cmdlet: Search store backends and search-capable plugins.""" from __future__ import annotations @@ -15,11 +15,11 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin from SYS.logger import log, debug, debug_panel from SYS.payload_builders import build_file_result_payload, normalize_file_extension -from ProviderCore.registry import get_search_plugin, list_search_plugins +from ProviderCore.registry import get_plugin_with_capability, list_plugins_with_capability from SYS.rich_display import ( - show_provider_config_panel, + show_plugin_config_panel, show_store_config_panel, - show_available_providers_panel, + show_available_plugins_panel, ) from SYS.database import insert_worker, update_worker, append_worker_stdout from SYS.item_accessors import get_extension_field, get_int_field, get_result_title @@ -164,13 +164,13 @@ def _summarize_worker_results(results: Sequence[Dict[str, Any]], preview_limit: class search_file(Cmdlet): - """Class-based search-file cmdlet for searching storage backends.""" + """Class-based search-file cmdlet for searching backends and providers.""" def __init__(self) -> None: super().__init__( name="search-file", - summary="Search storage backends (Hydrus) or external plugins (via -plugin).", - usage="search-file [-query ] [-store BACKEND] [-limit N] [-plugin NAME]", + summary="Search configured store backends or search-capable plugins.", + usage="search-file [-query ] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]", arg=[ CmdletArg( "limit", @@ -178,6 +178,7 @@ class search_file(Cmdlet): description="Limit results (default: 100)" ), SharedArgs.STORE, + SharedArgs.INSTANCE, SharedArgs.QUERY, SharedArgs.PLUGIN, CmdletArg( @@ -187,8 +188,10 @@ class search_file(Cmdlet): ), ], detail=[ - "Search across storage backends: Hydrus instances", - "Use -store to search a specific backend by name", + "Search across configured store backends or plugin providers.", + "Use -store to target a specific store backend by name.", + "Use -plugin with -instance to target a named provider config.", + "In plugin mode, -store is kept as a compatibility alias for -instance .", "URL search: url:* (any URL) or url: (URL substring)", "Extension search: ext: (e.g., ext:png)", "Hydrus-style extension: system:filetype = png", @@ -207,6 +210,7 @@ class search_file(Cmdlet): "", "Plugin search (-plugin):", "search-file -plugin youtube 'tutorial' # Search YouTube plugin", + "search-file -plugin ftp -instance work '*' # Search a named FTP/SCP plugin instance", "search-file -plugin alldebrid '*' # List AllDebrid magnets", "search-file -plugin alldebrid -open 123 '*' # Show files for a magnet", ], @@ -1451,6 +1455,7 @@ class search_file(Cmdlet): self, *, plugin_name: str, + instance_name: Optional[str], query: str, limit: int, limit_set: bool, @@ -1475,15 +1480,15 @@ class search_file(Cmdlet): log("Error: search-file -plugin requires both plugin and query", file=sys.stderr) log(f"Usage: {self.usage}", file=sys.stderr) - providers_map = list_search_plugins(config) + providers_map = list_plugins_with_capability("search", config) available = [n for n, a in providers_map.items() if a] unconfigured = [n for n, a in providers_map.items() if not a] if unconfigured: - show_provider_config_panel(unconfigured) + show_plugin_config_panel(unconfigured) if available: - show_available_providers_panel(available) + show_available_plugins_panel(available) return 1 @@ -1496,7 +1501,7 @@ class search_file(Cmdlet): if hasattr(ctx_mod, "get_pipeline_state"): progress = ctx_mod.get_pipeline_state().live_progress - provider = get_search_plugin(plugin_name, config) + provider = get_plugin_with_capability(plugin_name, "search", config) if not provider: if progress: try: @@ -1504,12 +1509,12 @@ class search_file(Cmdlet): except Exception: pass - show_provider_config_panel([plugin_name]) + show_plugin_config_panel([plugin_name]) - providers_map = list_search_plugins(config) + providers_map = list_plugins_with_capability("search", config) available = [n for n, a in providers_map.items() if a] if available: - show_available_providers_panel(available) + show_available_plugins_panel(available) return 1 worker_id = str(uuid.uuid4()) @@ -1542,6 +1547,8 @@ class search_file(Cmdlet): normalized_query = (normalized_query or "").strip() query = normalized_query or "*" search_filters = dict(provider_filters or {}) + if instance_name and not search_filters.get("instance"): + search_filters["instance"] = str(instance_name).strip() # Dynamic table generation via provider table_title = provider.get_table_title(query, search_filters).strip().rstrip(":") @@ -1564,6 +1571,7 @@ class search_file(Cmdlet): "search-file provider request", [ ("provider", plugin_name), + ("instance", search_filters.get("instance") or ""), ("query", query), ("limit", limit), ("filters", search_filters or ""), @@ -1581,7 +1589,7 @@ class search_file(Cmdlet): border_style="cyan", ) - # Allow providers to apply provider-specific UX transforms (e.g. auto-expansion) + # Allow plugins to apply plugin-specific UX transforms (e.g. auto-expansion) try: post = getattr(provider, "postprocess_search_results", None) if callable(post) and isinstance(results, list): @@ -1737,6 +1745,10 @@ class search_file(Cmdlet): f.lower() for f in (flag_registry.get("store") or {"-store", "--store"}) } + instance_flags = { + f.lower() + for f in (flag_registry.get("instance") or {"-instance", "--instance"}) + } limit_flags = { f.lower() for f in (flag_registry.get("limit") or {"-limit", "--limit"}) @@ -1753,6 +1765,7 @@ class search_file(Cmdlet): # Parse arguments query = "" storage_backend: Optional[str] = None + instance_name: Optional[str] = None plugin_name: Optional[str] = None open_id: Optional[int] = None limit = 100 @@ -1773,6 +1786,10 @@ class search_file(Cmdlet): plugin_name = args_list[i + 1] i += 2 continue + if low in instance_flags and i + 1 < len(args_list): + instance_name = args_list[i + 1] + i += 2 + continue if low in open_flags and i + 1 < len(args_list): try: open_id = int(args_list[i + 1]) @@ -1804,8 +1821,11 @@ class search_file(Cmdlet): query = query.strip() if plugin_name: + if storage_backend and not instance_name: + instance_name = storage_backend return self._run_plugin_search( plugin_name=plugin_name, + instance_name=instance_name, query=query, limit=limit, limit_set=limit_set, diff --git a/cmdnat/matrix.py b/cmdnat/matrix.py index 5bd7ffb..e454e40 100644 --- a/cmdnat/matrix.py +++ b/cmdnat/matrix.py @@ -493,7 +493,7 @@ def _maybe_download_hydrus_file(item: Any, """ try: from SYS.config import get_hydrus_access_key, get_hydrus_url - from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file + from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient, download_hydrus_file # Prefer per-item Hydrus instance name when it matches a configured instance. store_name = None diff --git a/cmdnat/pipe.py b/cmdnat/pipe.py index c9176eb..a187c06 100644 --- a/cmdnat/pipe.py +++ b/cmdnat/pipe.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta from urllib.parse import urlparse, parse_qs from pathlib import Path from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args -from ProviderCore.registry import get_plugin_for_url +from ProviderCore.registry import get_plugin, get_plugin_for_url from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream from SYS.result_table import Table from MPV.mpv_ipc import MPV @@ -32,6 +32,13 @@ _IPV4_RE = re.compile(r"^\d+\.\d+\.\d+\.\d+$") _MPD_PATH_RE = re.compile(r"\.mpd($|\?)") +def _get_hydrus_provider(config: Optional[Dict[str, Any]] = None) -> Any: + try: + return get_plugin("hydrusnetwork", config or {}) + except Exception: + return None + + def _repo_root() -> Path: try: return Path(__file__).resolve().parent.parent @@ -575,20 +582,51 @@ def _send_ipc_command(command: Dict[str, Any], silent: bool = False, wait: bool return None -def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]: +def _extract_store_and_hash( + item: Any, + *, + config: Optional[Dict[str, Any]] = None, +) -> tuple[Optional[str], Optional[str]]: store: Optional[str] = None file_hash: Optional[str] = None + targets: list[str] = [] + + def _add_target(value: Any) -> None: + text = str(value or "").strip() + if text: + targets.append(text) try: if isinstance(item, dict): store = item.get("store") file_hash = item.get("hash") or item.get("file_hash") + _add_target(item.get("path")) + _add_target(item.get("url")) + _add_target(item.get("filename")) + metadata = item.get("full_metadata") or item.get("metadata") else: store = getattr(item, "store", None) file_hash = getattr(item, "hash", None) or getattr(item, "file_hash", None) + _add_target(getattr(item, "path", None)) + _add_target(getattr(item, "url", None)) + metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None) except Exception: store = None file_hash = None + metadata = None + + if isinstance(metadata, dict): + try: + if not store: + store = metadata.get("store") + if not file_hash or str(file_hash).strip().lower() in {"unknown", "none", "n/a", "na"}: + file_hash = metadata.get("hash") or metadata.get("hash_hex") + _add_target(metadata.get("path")) + _add_target(metadata.get("url")) + _add_target(metadata.get("selection_url")) + _add_target(metadata.get("hydrus_url")) + except Exception: + pass try: store = str(store).strip() if store else None @@ -600,17 +638,37 @@ def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]: except Exception: file_hash = None + hydrus_provider = _get_hydrus_provider(config) + if hydrus_provider is not None: + normalized_store = None + try: + if store and hydrus_provider.is_store_name(store): + normalized_store = store + except Exception: + normalized_store = None + + for target in targets: + try: + parsed_store, parsed_hash = hydrus_provider.parse_hydrus_url(target) + except Exception: + parsed_store, parsed_hash = None, "" + if parsed_hash and not file_hash: + file_hash = parsed_hash + if parsed_store: + normalized_store = parsed_store + + if normalized_store: + store = normalized_store + elif store and store.upper() in {"PATH", "LOCAL", "UNKNOWN"}: + store = None + if not file_hash: try: - text = None - if isinstance(item, dict): - text = item.get("path") or item.get("url") or item.get("filename") - else: - text = getattr(item, "path", None) or getattr(item, "url", None) - if text: + for text in targets: m = _SHA256_RE.search(str(text).lower()) if m: file_hash = m.group(0) + break except Exception: pass @@ -681,6 +739,8 @@ def _prefetch_notes_async( set_notes_prefetch_pending(store, file_hash, True) registry = Store(cfg, suppress_debug=True) + if not registry.is_available(str(store)): + return backend = registry[str(store)] notes = backend.get_note(str(file_hash), config=cfg) or {} store_cached_notes(store, file_hash, notes) @@ -718,7 +778,7 @@ def _schedule_notes_prefetch(items: Sequence[Any], config: Optional[Dict[str, An seen: set[str] = set() scheduled = 0 for item in items or []: - store, file_hash = _extract_store_and_hash(item) + store, file_hash = _extract_store_and_hash(item, config=config) if not store or not file_hash: continue key = f"{store.lower()}:{file_hash}" @@ -789,7 +849,12 @@ def _extract_target_from_memory_uri(text: str) -> Optional[str]: return None -def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional[str]: +def _find_hydrus_instance_for_hash( + hash_str: str, + file_storage: Any, + *, + config: Optional[Dict[str, Any]] = None, +) -> Optional[str]: """Find which Hydrus instance serves a specific file hash. Args: @@ -799,27 +864,29 @@ def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional Returns: Instance name (e.g., 'home') or None if not found """ - # Query each Hydrus backend to see if it has this file - for backend_name in file_storage.list_backends(): - backend = file_storage[backend_name] - # Check if this is a Hydrus backend by checking class name - backend_class = type(backend).__name__ - if backend_class != "HydrusNetwork": - continue + hydrus_provider = _get_hydrus_provider(config) + if hydrus_provider is None: + return None - try: - # Query metadata to see if this instance has the file - metadata = backend.get_metadata(hash_str) - if metadata: - return backend_name - except Exception: - # This instance doesn't have the file or had an error - continue + try: + for backend_name, _backend in hydrus_provider.iter_backends(): + try: + if hydrus_provider.hash_exists(hash_str, store_name=str(backend_name)): + return str(backend_name) + except Exception: + continue + except Exception: + return None return None -def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]: +def _find_hydrus_instance_by_url( + url: str, + file_storage: Any, + *, + config: Optional[Dict[str, Any]] = None, +) -> Optional[str]: """Find which Hydrus instance matches a given URL. Args: @@ -829,31 +896,13 @@ def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]: Returns: Instance name (e.g., 'home') or None if not found """ - from urllib.parse import urlparse - - parsed_target = urlparse(url) - target_netloc = parsed_target.netloc.lower() - - # Check each Hydrus backend's URL - for backend_name in file_storage.list_backends(): - backend = file_storage[backend_name] - backend_class = type(backend).__name__ - if backend_class != "HydrusNetwork": - continue - - # Get the backend's base URL from its client - try: - backend_url = backend._client.base_url - parsed_backend = urlparse(backend_url) - backend_netloc = parsed_backend.netloc.lower() - - # Match by netloc (host:port) - if target_netloc == backend_netloc: - return backend_name - except Exception: - continue - - return None + hydrus_provider = _get_hydrus_provider(config) + if hydrus_provider is None: + return None + try: + return hydrus_provider.match_store_name_for_url(url) + except Exception: + return None def _normalize_playlist_path(text: Optional[str]) -> Optional[str]: @@ -891,7 +940,8 @@ def _normalize_playlist_path(text: Optional[str]) -> Optional[str]: def _infer_store_from_playlist_item( item: Dict[str, Any], - file_storage: Optional[Any] = None + file_storage: Optional[Any] = None, + config: Optional[Dict[str, Any]] = None, ) -> str: """Infer a friendly store label from an MPV playlist entry. @@ -915,7 +965,7 @@ def _infer_store_from_playlist_item( # If we have file_storage, query each Hydrus instance to find which one has this hash if file_storage: hash_str = target.lower() - hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) + hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config) if hydrus_instance: return hydrus_instance return "hydrus" @@ -929,7 +979,7 @@ def _infer_store_from_playlist_item( hash_match = _SHA256_RE.search(target.lower()) if hash_match: hash_str = hash_match.group(0) - hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) + hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config) if hydrus_instance: return hydrus_instance return "hydrus" @@ -968,11 +1018,11 @@ def _infer_store_from_playlist_item( hash_match = _HASH_QUERY_RE.search(target.lower()) if hash_match: hash_str = hash_match.group(1) - hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) + hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config) if hydrus_instance: return hydrus_instance # If no hash in URL, try matching the base URL to configured instances - hydrus_instance = _find_hydrus_instance_by_url(target, file_storage) + hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config) if hydrus_instance: return hydrus_instance return "hydrus" @@ -982,10 +1032,10 @@ def _infer_store_from_playlist_item( hash_match = _HASH_QUERY_RE.search(target.lower()) if hash_match: hash_str = hash_match.group(1) - hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) + hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config) if hydrus_instance: return hydrus_instance - hydrus_instance = _find_hydrus_instance_by_url(target, file_storage) + hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config) if hydrus_instance: return hydrus_instance return "hydrus" @@ -1320,6 +1370,16 @@ def _get_playable_path( elif isinstance(item, str): path = item + if not store or not file_hash or str(file_hash).strip().lower() == "unknown": + try: + extracted_store, extracted_hash = _extract_store_and_hash(item, config=config) + except Exception: + extracted_store, extracted_hash = None, None + if extracted_store and not store: + store = extracted_store + if extracted_hash and (not file_hash or str(file_hash).strip().lower() == "unknown"): + file_hash = extracted_hash + # Debug: show incoming values try: debug(f"_get_playable_path: store={store}, path={path}, hash={file_hash}") @@ -1384,6 +1444,7 @@ def _get_playable_path( # - MPV IPC pipe (transport) # - PipeObject (pipeline data) backend_target_resolved = False + hydrus_provider = _get_hydrus_provider(config) if store and file_hash and file_hash != "unknown" and file_storage: try: backend = file_storage[store] @@ -1391,18 +1452,14 @@ def _get_playable_path( backend = None if backend is not None: - backend_class = type(backend).__name__ backend_target_resolved = True - # HydrusNetwork: build a playable API file URL without browser side-effects. - if backend_class == "HydrusNetwork": + # Hydrus playback should resolve via the provider so store aliases and URL building stay centralized. + if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(store)): try: - client = getattr(backend, "_client", None) - base_url = getattr(client, "url", None) - if base_url: - base_url = str(base_url).rstrip("/") - # Auth is provided via http-header-fields (set in _queue_items). - path = f"{base_url}/get_files/file?hash={file_hash}" + resolved_path = hydrus_provider.build_file_url(file_hash, store_name=str(store)) + if resolved_path: + path = resolved_path except Exception as e: debug( f"Error building Hydrus URL from store '{store}': {e}", @@ -1570,32 +1627,43 @@ def _queue_items( item_store = None if isinstance(item, dict): item_store = item.get("store") + metadata = item.get("full_metadata") or item.get("metadata") else: item_store = getattr(item, "store", None) + metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None) + + if not item_store and isinstance(metadata, dict): + item_store = metadata.get("store") + + if not item_store: + item_store, _ = _extract_store_and_hash(item, config=config) if item_store: item_store_name = str(item_store).strip() or None - if item_store and file_storage: - try: - backend = file_storage[str(item_store)] - except Exception: - backend = None - - if backend is not None and type(backend).__name__ == "HydrusNetwork": - client = getattr(backend, "_client", None) - base_url = getattr(client, "url", None) - key = getattr(client, "access_key", None) - if base_url: - effective_hydrus_url = str(base_url).rstrip("/") - if key: - effective_hydrus_header = ( - f"Hydrus-Client-API-Access-Key: {str(key).strip()}" - ) - effective_ytdl_opts = _build_ytdl_options( - config, - effective_hydrus_header + if item_store_name and file_storage: + hydrus_provider = _get_hydrus_provider(config) + if hydrus_provider is not None: + try: + client = hydrus_provider.get_client( + store_name=item_store_name, + allow_default=False, ) + except Exception: + client = None + if client is not None: + base_url = getattr(client, "url", None) + key = getattr(client, "access_key", None) + if base_url: + effective_hydrus_url = str(base_url).rstrip("/") + if key: + effective_hydrus_header = ( + f"Hydrus-Client-API-Access-Key: {str(key).strip()}" + ) + effective_ytdl_opts = _build_ytdl_options( + config, + effective_hydrus_header + ) except Exception: pass @@ -1608,6 +1676,10 @@ def _queue_items( f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}" ) + hydrus_target = bool( + effective_hydrus_header and _is_hydrus_path(str(target), effective_hydrus_url) + ) + norm_key = _normalize_playlist_path(target) or str(target).strip().lower() if norm_key in existing_targets or norm_key in new_targets: debug(f"Skipping duplicate playlist entry: {title or target}") @@ -1616,10 +1688,11 @@ def _queue_items( # Use memory:// M3U hack to pass title to MPV. # Avoid this for probable ytdl URLs because it can prevent the hook from triggering. - if title and not _is_probable_ytdl_url(target): + safe_title = title.replace("\n", " ").replace("\r", "") if title else None + if title and hydrus_target: + target_to_send = target + elif title and not _is_probable_ytdl_url(target): # Sanitize title for M3U (remove newlines) - safe_title = title.replace("\n", " ").replace("\r", "") - # Carry the store name for hash URLs so MPV.lyric can resolve the backend. # This is especially important for local file-server URLs like /get_files/file?hash=... target_for_m3u = target @@ -1646,15 +1719,14 @@ def _queue_items( # so MPV.lyric can resolve the correct backend for notes. if mode == "replace": try: - s, h = _extract_store_and_hash(item) + s, h = _extract_store_and_hash(item, config=config) _set_mpv_item_context(s, h) except Exception: pass # If this is a Hydrus path, set header property and yt-dlp headers before loading. # Use the real target (not the memory:// wrapper) for detection. - if effective_hydrus_header and _is_hydrus_path(str(target), - effective_hydrus_url): + if hydrus_target: header_cmd = { "command": ["set_property", @@ -1683,10 +1755,18 @@ def _queue_items( except Exception: pass + load_options: Dict[str, str] = {} + if hydrus_target and command_name == "loadfile": + load_options["ytdl"] = "no" + if safe_title: + load_options["force-media-title"] = safe_title + + command_args: List[Any] = [command_name, target_to_send, mode] + if load_options: + command_args.extend([-1, load_options]) + cmd = { - "command": [command_name, - target_to_send, - mode], + "command": command_args, "request_id": 200 } try: @@ -2159,7 +2239,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # Prefer the store/hash from the piped item when auto-playing. try: - s, h = _extract_store_and_hash(items_to_add[0]) + s, h = _extract_store_and_hash(items_to_add[0], config=config) _set_mpv_item_context(s, h) except Exception: pass @@ -2293,7 +2373,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: else: # Play item try: - s, h = _extract_store_and_hash(item) + s, h = _extract_store_and_hash(item, config=config) _set_mpv_item_context(s, h) except Exception: pass @@ -2376,26 +2456,17 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # Extract the real path/URL from memory:// wrapper if present real_path = _extract_target_from_memory_uri(filename) or filename - # Try to extract hash from the path/URL - file_hash = None - store_name = None + # Try to extract hash/store from the path/URL via provider-aware normalization. + store_name, file_hash = _extract_store_and_hash( + { + "path": real_path, + "title": title, + }, + config=config, + ) - # Check if it's a Hydrus URL - if "get_files/file" in real_path or "hash=" in real_path: - # Extract hash from Hydrus URL - hash_match = _HASH_QUERY_RE.search(real_path.lower()) - if hash_match: - file_hash = hash_match.group(1) - # Try to find which Hydrus instance has this file - if file_storage: - store_name = _find_hydrus_instance_for_hash( - file_hash, - file_storage - ) - if not store_name: - store_name = "hydrus" # Check if it's a hash-based local file - elif real_path: + if not file_hash and real_path: # Try to extract hash from filename (e.g., C:\path\1e8c46...a1b2.mp4) path_obj = Path(real_path) stem = path_obj.stem # filename without extension @@ -2407,7 +2478,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: if not store_name: store_name = _infer_store_from_playlist_item( item, - file_storage=file_storage + file_storage=file_storage, + config=config, ) # Build PipeObject with proper metadata @@ -2651,7 +2723,7 @@ def _start_mpv( # target change (the helper may start before playback begins). try: if items: - s, h = _extract_store_and_hash(items[0]) + s, h = _extract_store_and_hash(items[0], config=config) _set_mpv_item_context(s, h) except Exception: pass diff --git a/docs/ftp_plugin_tutorial.md b/docs/ftp_plugin_tutorial.md index 3a393cd..be3264d 100644 --- a/docs/ftp_plugin_tutorial.md +++ b/docs/ftp_plugin_tutorial.md @@ -2,11 +2,11 @@ This walkthrough adds a real bundled `ftp` plugin so users can: -- run `search-file -plugin ftp ...` +- run `search-file -plugin ftp -instance ...` - browse remote folders as result tables - select file rows to `download-file` - pipe selected file rows into `add-file` -- upload local files with `add-file -plugin ftp` +- upload local files with `add-file -plugin ftp -instance ` The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). @@ -20,14 +20,14 @@ The FTP plugin demonstrates the main provider hooks that matter for a storage-st - `selector()` turns folder rows into a follow-up table when the user runs `@N`. - `download()` and `download_url()` fetch FTP files into `download-file` output paths. - `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first. -- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server. +- `upload()` lets `add-file -plugin ftp -instance -path ...` push a local file to the configured FTP server. ## Example Config -Add an FTP provider block to your config: +Add one or more named FTP provider instances to your config: ```toml -[provider.ftp] +[provider.ftp.work] host = "ftp.example.com" port = 21 username = "demo" @@ -37,11 +37,20 @@ tls = false passive = true timeout = 20 search_depth = 1 + +[provider.ftp.archive] +host = "archive.example.com" +port = 2121 +username = "archive-bot" +password = "secret" +base_path = "/dropbox" +tls = true ``` Notes: -- `host` is the only required field for the plugin to validate. +- `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`. +- `host` is the only required field for each instance to validate. - `username` defaults to `anonymous` and `password` defaults to `anonymous@`. - `base_path` is both the default search root and the upload target directory. - `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default. @@ -51,25 +60,25 @@ Notes: Basic listing from the configured base path: ```powershell -search-file -plugin ftp "*" +search-file -plugin ftp -instance work "*" ``` Search by filename fragment: ```powershell -search-file -plugin ftp "invoice" +search-file -plugin ftp -instance work "invoice" ``` Search a different subtree and recurse deeper: ```powershell -search-file -plugin ftp "path:/pub depth:2 invoice" +search-file -plugin ftp -instance work "path:/pub depth:2 invoice" ``` Filter to folders only: ```powershell -search-file -plugin ftp "path:/pub type:folder *" +search-file -plugin ftp -instance work "path:/pub type:folder *" ``` The plugin returns rows with explicit columns for name, type, directory, size, and modification time. @@ -79,20 +88,20 @@ The plugin returns rows with explicit columns for name, type, directory, size, a Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory: ```powershell -search-file -plugin ftp "*" +search-file -plugin ftp -instance work "*" @2 ``` File rows carry an explicit row action: ```powershell -download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf +download-file -plugin ftp -instance work -url ftp://ftp.example.com/incoming/report.pdf ``` That means plain `@N` on a file row downloads it immediately: ```powershell -search-file -plugin ftp "report" +search-file -plugin ftp -instance work "report" @1 ``` @@ -101,14 +110,14 @@ search-file -plugin ftp "report" If you want the downloaded file in a specific local directory: ```powershell -search-file -plugin ftp "report" +search-file -plugin ftp -instance work "report" @1 | download-file -path C:\Downloads ``` If you want to ingest the selected FTP file into a configured store backend: ```powershell -search-file -plugin ftp "report" +search-file -plugin ftp -instance work "report" @1 | add-file -store tutorial ``` @@ -117,23 +126,24 @@ Why this works: - the file row advertises a `download-file` row action - the pipeline auto-inserts that download before `add-file` - the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion +- file rows also carry the chosen `instance`, so selection replay and `@N | add-file ...` keep the same FTP target ## Upload Flow -Uploading uses the same provider name, but through `add-file -plugin ftp`: +Uploading uses the same provider name, but through `add-file -plugin ftp -instance `: ```powershell -add-file -plugin ftp -path C:\Media\report.pdf +add-file -plugin ftp -instance archive -path C:\Media\report.pdf ``` -That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result. +That sends the file to the selected instance's FTP `base_path` and returns the FTP URL as the uploaded result. ## Why The Row Metadata Matters The critical part of this plugin is the file-row metadata: -- file rows emit `_selection_args` as `['-url', '']` -- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '']` +- file rows emit `_selection_args` as `['-instance', '', '-url', '']` +- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-instance', '', '-url', '']` - folder rows do not emit a download action, so `selector()` can own drill-in behavior instead That split is what keeps these two user experiences compatible: @@ -155,10 +165,10 @@ The code is intentionally small and uses only Python stdlib pieces: ## Recommended Commands To Demo The Walkthrough ```powershell -search-file -plugin ftp "*" -search-file -plugin ftp "path:/incoming depth:2 *.pdf" +search-file -plugin ftp -instance work "*" +search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf" @1 @1 | download-file -path C:\Downloads @1 | add-file -store tutorial -add-file -plugin ftp -path C:\Media\report.pdf +add-file -plugin ftp -instance archive -path C:\Media\report.pdf ``` \ No newline at end of file diff --git a/docs/scp_plugin_tutorial.md b/docs/scp_plugin_tutorial.md index 87758a3..4b83ed7 100644 --- a/docs/scp_plugin_tutorial.md +++ b/docs/scp_plugin_tutorial.md @@ -11,16 +11,16 @@ The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py). The SCP plugin mirrors the FTP walkthrough, but on top of SSH: -- `search-file -plugin scp ...` lists remote files and folders over SFTP. +- `search-file -plugin scp -instance ...` lists remote files and folders over SFTP. - plain `@N` on a folder drills into that directory. -- plain `@N` on a file runs `download-file -plugin scp -url ...`. +- plain `@N` on a file runs `download-file -plugin scp -instance -url ...`. - `@N | add-file -store ...` downloads first, then ingests the local temp file. -- `add-file -plugin scp -path ...` uploads a local file to the configured remote path. +- `add-file -plugin scp -instance -path ...` uploads a local file to the configured remote path. ## Example Config ```toml -[provider.scp] +[provider.scp.work] host = "ssh.example.com" port = 22 username = "deploy" @@ -31,11 +31,20 @@ timeout = 20 search_depth = 1 allow_agent = true look_for_keys = true + +[provider.scp.archive] +host = "ssh-archive.example.com" +port = 2222 +username = "archive" +key_path = "C:/Users/Admin/.ssh/archive_ed25519" +base_path = "/srv/archive" +timeout = 20 ``` Notes: -- `host` and `username` are required for the plugin to validate. +- `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`. +- `host` and `username` are required for each instance to validate. - You can use password auth, key auth, or both. - `base_path` is both the default search root and the default upload directory. @@ -44,25 +53,25 @@ Notes: List the configured base path: ```powershell -search-file -plugin scp "*" +search-file -plugin scp -instance work "*" ``` Search by filename: ```powershell -search-file -plugin scp "invoice" +search-file -plugin scp -instance work "invoice" ``` Search another subtree with deeper recursion: ```powershell -search-file -plugin scp "path:/srv/files/releases depth:2 *.zip" +search-file -plugin scp -instance work "path:/srv/files/releases depth:2 *.zip" ``` Show only folders: ```powershell -search-file -plugin scp "path:/srv/files type:folder *" +search-file -plugin scp -instance work "path:/srv/files type:folder *" ``` ## Selection Flow @@ -70,21 +79,21 @@ search-file -plugin scp "path:/srv/files type:folder *" Folder rows are navigation rows: ```powershell -search-file -plugin scp "*" +search-file -plugin scp -instance work "*" @2 ``` File rows carry an explicit row action, so terminal selection downloads directly: ```powershell -search-file -plugin scp "report" +search-file -plugin scp -instance work "report" @1 ``` That expands to the equivalent of: ```powershell -download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf +download-file -plugin scp -instance work -url scp://ssh.example.com/srv/files/report.pdf ``` ## Download And Add-File Flow @@ -92,14 +101,14 @@ download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf Download into a local folder: ```powershell -search-file -plugin scp "report" +search-file -plugin scp -instance work "report" @1 | download-file -path C:\Downloads ``` Ingest a selected remote file into a configured store backend: ```powershell -search-file -plugin scp "report" +search-file -plugin scp -instance work "report" @1 | add-file -store tutorial ``` @@ -108,13 +117,14 @@ Why this works: - file rows advertise `_selection_action` for `download-file` - `add-file` selection replay inserts that provider download stage before ingest - the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows +- file rows also carry the chosen `instance`, so replay stays bound to the same SSH target ## Upload Flow Upload a local file to the configured remote `base_path`: ```powershell -add-file -plugin scp -path C:\Media\report.pdf +add-file -plugin scp -instance archive -path C:\Media\report.pdf ``` ## Implementation Notes @@ -127,10 +137,10 @@ The plugin uses SFTP for directory listing because SCP itself is a transfer prot ## Recommended Demo Commands ```powershell -search-file -plugin scp "*" -search-file -plugin scp "path:/srv/files depth:2 *.zip" +search-file -plugin scp -instance work "*" +search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip" @1 @1 | download-file -path C:\Downloads @1 | add-file -store tutorial -add-file -plugin scp -path C:\Media\report.pdf +add-file -plugin scp -instance archive -path C:\Media\report.pdf ``` \ No newline at end of file diff --git a/plugins/README.md b/plugins/README.md index be88913..209545d 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -52,8 +52,9 @@ class MyPlugin(Provider): Bundled walkthrough: +- Providers can now expose named config instances under `provider..` and cmdlets can target them with `-instance `; plugin-mode `-store ` remains a compatibility alias while store-backed flows still use real backend stores. - The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). -- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads. +- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance `, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance ` uploads. - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). -- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads. -- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly. \ No newline at end of file +- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance `, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance ` uploads. +- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives alongside it in [plugins/hydrusnetwork/api.py](plugins/hydrusnetwork/api.py), while [API/HydrusNetwork.py](API/HydrusNetwork.py) remains a compatibility shim. The provider delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly. \ No newline at end of file diff --git a/plugins/alldebrid/__init__.py b/plugins/alldebrid/__init__.py index 37aad1c..0e0e582 100644 --- a/plugins/alldebrid/__init__.py +++ b/plugins/alldebrid/__init__.py @@ -540,7 +540,7 @@ def download_magnet( def expand_folder_item( item: Any, - get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]], + get_plugin: Optional[Callable[[str, Dict[str, Any]], Any]], config: Dict[str, Any], ) -> Tuple[List[Any], Optional[str]]: table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table") @@ -564,10 +564,10 @@ def expand_folder_item( except Exception: magnet_id = None - if magnet_id is None or get_search_plugin is None: + if magnet_id is None or get_plugin is None: return [], None - plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None + plugin = get_plugin("alldebrid", config) if get_plugin else None if plugin is None: return [], None @@ -1774,6 +1774,80 @@ class AllDebrid(TableProviderMixin, Provider): return True + def show_selection_details( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + source_command: str = "", + table_type: str = "", + table_metadata: Optional[Dict[str, Any]] = None, + **_kwargs: Any, + ) -> bool: + _ = table_type + _item, payload, meta = self.resolve_selection_detail_subject( + selected_items, + stage_is_last=stage_is_last, + source_command=source_command, + require_media_kind="file", + ) + if not isinstance(payload, dict): + return False + + title = str(payload.get("title") or meta.get("name") or "").strip() or "AllDebrid Item" + magnet_name = str(meta.get("magnet_name") or payload.get("detail") or "").strip() + magnet_id = meta.get("magnet_id") + relpath = str(meta.get("relpath") or "").strip() + direct_url = str(payload.get("path") or "").strip() + selection_url = "" + action = meta.get("_selection_action") or meta.get("selection_action") + if isinstance(action, (list, tuple)): + action_tokens = [str(x) for x in action if x is not None] + for idx, token in enumerate(action_tokens): + if str(token).strip().lower() in {"-url", "--url"} and idx + 1 < len(action_tokens): + selection_url = str(action_tokens[idx + 1] or "").strip() + break + + try: + from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view + except Exception: + return super().show_selection_details( + selected_items, + ctx=ctx, + stage_is_last=stage_is_last, + source_command=source_command, + table_type=table_type, + table_metadata=table_metadata, + ) + + detail_metadata = prepare_detail_metadata( + payload, + title=title, + store=self.name, + path=direct_url or None, + tags=meta.get("tag") or meta.get("tags"), + extra_fields={ + "Plugin": self.name, + "Magnet": magnet_name or None, + "Magnet ID": magnet_id, + "Relative Path": relpath or None, + "View": str(meta.get("provider_view") or meta.get("view") or (table_metadata or {}).get("view") or "").strip() or None, + "Direct Url": direct_url or None, + "Selection Url": selection_url or None, + }, + ) + + return render_selection_detail_view( + ctx=ctx, + item=payload, + title=f"AllDebrid Item: {title}", + metadata=detail_metadata, + table_name=self.name, + detail_order=["Title", "Store", "Magnet", "Magnet ID", "Relative Path", "View", "Path", "File", "Folder", "ID", "Direct URL", "Selection URL", "Plugin"], + value_case="preserve", + ) + try: from SYS.result_table_adapters import register_plugin diff --git a/plugins/example_provider.py b/plugins/example_provider.py index 30158b3..d3bc824 100644 --- a/plugins/example_provider.py +++ b/plugins/example_provider.py @@ -1,7 +1,259 @@ -"""Plugin-namespace import shim for the strict adapter example module. +"""Example plugin that uses the new `ResultTable` API. -This keeps example docs pointing at the plugin namespace while the original -implementation remains in ``Provider.example_provider`` for compatibility. +This module demonstrates a minimal provider adapter that yields `ResultModel` +instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer +(`render_table`) for demonstration. + +Run this to see sample output: + python -m Provider.example_provider + +Example usage (piped selector): + plugin-table -plugin example -sample | select -select 1 | add-file -store default """ +from __future__ import annotations -from Provider.example_provider import * # noqa: F401,F403 +from pathlib import Path +from typing import Any, Dict, Iterable, List + +from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column + + +SAMPLE_ITEMS = [ + { + "name": "Book of Awe.pdf", + "path": "sample/Book of Awe.pdf", + "ext": "pdf", + "size": 1024000, + "source": "example", + }, + { + "name": "Song of Joy.mp3", + "path": "sample/Song of Joy.mp3", + "ext": "mp3", + "size": 5120000, + "source": "example", + }, + { + "name": "Cover Image.jpg", + "path": "sample/Cover Image.jpg", + "ext": "jpg", + "size": 20480, + "source": "example", + }, +] + + +def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]: + """Convert provider-specific items into `ResultModel` instances. + + This adapter enforces the strict API requirement: it yields only + `ResultModel` instances (no legacy dict objects). + """ + for it in items: + title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "") + yield ResultModel( + title=str(title), + path=str(it.get("path")) if it.get("path") else None, + ext=str(it.get("ext")) if it.get("ext") else None, + size_bytes=int(it.get("size")) if it.get("size") is not None else None, + metadata=dict(it), + source=str(it.get("source")) if it.get("source") else "example", + ) + + +# Columns are intentionally *not* mandated. Create a factory that inspects +# sample rows and builds only columns that make sense for the provider data. +from SYS.result_table_api import metadata_column + + +def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: + cols: List[ColumnSpec] = [title_column()] + + # If any row has an extension, include Ext column + if any(getattr(r, "ext", None) for r in rows): + cols.append(ext_column()) + + # If any row has size, include Size column + if any(getattr(r, "size_bytes", None) for r in rows): + cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v))) + + # Add any top-level metadata keys discovered (up to 3) as optional columns + seen_keys = [] + for r in rows: + for k in (r.metadata or {}).keys(): + if k in ("name", "title", "path"): + continue + if k not in seen_keys: + seen_keys.append(k) + if len(seen_keys) >= 3: + break + if len(seen_keys) >= 3: + break + + for k in seen_keys: + cols.append(metadata_column(k)) + + return cols + + +# Selection function: cmdlets rely on this to build selector args when the user +# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer +# -path if available, otherwise fall back to -title. +def selection_fn(row: ResultModel) -> List[str]: + if row.path: + return ["-path", row.path] + return ["-title", row.title] + + +# Register the plugin with the registry so callers can discover it by name +from SYS.result_table_adapters import register_plugin +register_plugin( + "example", + adapter, + columns=columns_factory, + selection_fn=selection_fn, + metadata={"description": "Example provider demonstrating dynamic columns and selectors"}, +) + + +def _format_size(size: Any) -> str: + try: + s = int(size) + except Exception: + return "" + if s >= 1024 ** 3: + return f"{s / (1024 ** 3):.2f} GB" + if s >= 1024 ** 2: + return f"{s / (1024 ** 2):.2f} MB" + if s >= 1024: + return f"{s / 1024:.2f} KB" + return f"{s} B" + + +def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str: + """Render a simple ASCII table of `rows` using `columns`. + + This is intentionally very small and dependency-free for demonstration. + Renderers in the project should implement the `Renderer` protocol. + """ + rows = list(rows) + + # Build cell matrix (strings) + matrix: List[List[str]] = [] + for r in rows: + cells: List[str] = [] + for col in columns: + raw = col.extractor(r) + if col.format_fn: + try: + cell = col.format_fn(raw) + except Exception: + cell = str(raw or "") + else: + cell = str(raw or "") + cells.append(cell) + matrix.append(cells) + + # Compute column widths as max(header, content) + headers = [c.header for c in columns] + widths = [len(h) for h in headers] + for row_cells in matrix: + for i, cell in enumerate(row_cells): + widths[i] = max(widths[i], len(cell)) + + # Helper to format a row + def fmt_row(cells: List[str]) -> str: + return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) + + lines: List[str] = [] + lines.append(fmt_row(headers)) + lines.append("-+-".join("-" * w for w in widths)) + for row_cells in matrix: + lines.append(fmt_row(row_cells)) + + return "\n".join(lines) + + +# Rich-based renderer (returns a Rich Table renderable) +def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]): + """Render rows as a `rich.table.Table` for terminal output. + + Returns the Table object; callers may `Console.print(table)` to render. + """ + try: + from rich.table import Table as RichTable + except Exception as exc: # pragma: no cover - rare if rich missing + raise RuntimeError("rich is required for rich renderer") from exc + + table = RichTable(show_header=True, header_style="bold") + for col in columns: + table.add_column(col.header) + + for r in rows: + cells: List[str] = [] + for col in columns: + raw = col.extractor(r) + if col.format_fn: + try: + cell = col.format_fn(raw) + except Exception: + cell = str(raw or "") + else: + cell = str(raw or "") + cells.append(cell) + table.add_row(*cells) + + return table + + +def demo() -> None: + rows = list(adapter(SAMPLE_ITEMS)) + table = render_table_rich(rows, columns_factory(rows)) + try: + from rich.console import Console + except Exception: + # Fall back to plain printing if rich is not available + print("Example provider output:") + print(render_table(rows, columns_factory(rows))) + return + + console = Console() + console.print("Example provider output:") + console.print(table) + + +def demo_with_selection(idx: int = 0) -> None: + """Demonstrate how a cmdlet would use plugin registration and selection args. + + - Fetch the registered plugin by name + - Build rows via adapter + - Render the table + - Show the selection args for the chosen row; these are the args a cmdlet + would append when the user picks that row. + """ + from SYS.result_table_adapters import get_plugin + + provider = get_plugin("example") + rows = list(provider.adapter(SAMPLE_ITEMS)) + cols = provider.get_columns(rows) + + # Render + try: + from rich.console import Console + except Exception: + print(render_table(rows, cols)) + sel_args = provider.selection_args(rows[idx]) + print("Selection args for row", idx, "->", sel_args) + return + + console = Console() + console.print("Example provider output:") + console.print(render_table_rich(rows, cols)) + + # Selection args example + sel = provider.selection_args(rows[idx]) + console.print("Selection args for row", idx, "->", sel) + + +if __name__ == "__main__": + demo() diff --git a/plugins/ftp/__init__.py b/plugins/ftp/__init__.py index fe61fe8..4516813 100644 --- a/plugins/ftp/__init__.py +++ b/plugins/ftp/__init__.py @@ -12,18 +12,6 @@ from urllib.parse import quote, unquote, urlparse from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments -def _pick_provider_config(config: Any) -> Dict[str, Any]: - if not isinstance(config, dict): - return {} - provider = config.get("provider") - if not isinstance(provider, dict): - return {} - entry = provider.get("ftp") - if isinstance(entry, dict): - return entry - return {} - - def _coerce_bool(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value @@ -155,20 +143,54 @@ class FTP(Provider): def __init__(self, config: Optional[Dict[str, Any]] = None): super().__init__(config) - conf = _pick_provider_config(self.config) - self._host = str(conf.get("host") or "").strip() - self._tls = _coerce_bool(conf.get("tls"), False) - self._port = _coerce_int(conf.get("port"), 21) - self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous" - password_value = conf.get("password") - self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@" - self._passive = _coerce_bool(conf.get("passive"), True) - self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) - self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) - self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") + _instance_name, conf = self.resolve_plugin_instance() + defaults = self._settings_from_config(conf) + self._host = str(defaults.get("host") or "").strip() + self._tls = bool(defaults.get("tls")) + self._port = int(defaults.get("port") or 21) + self._username = str(defaults.get("username") or "anonymous").strip() or "anonymous" + self._password = str(defaults.get("password") or "anonymous@").strip() or "anonymous@" + self._passive = bool(defaults.get("passive")) + self._timeout = max(1, int(defaults.get("timeout") or 20)) + self._search_depth = max(0, int(defaults.get("search_depth") or 1)) + self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/") + + def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]: + entry = dict(conf or {}) + password_value = entry.get("password") + return { + "instance": str(instance_name or entry.get("_instance_name") or "").strip() or None, + "host": str(entry.get("host") or "").strip(), + "tls": _coerce_bool(entry.get("tls"), False), + "port": _coerce_int(entry.get("port"), 21), + "username": str(entry.get("username") or entry.get("user") or "anonymous").strip() or "anonymous", + "password": str(password_value).strip() if password_value not in (None, "") else "anonymous@", + "passive": _coerce_bool(entry.get("passive"), True), + "timeout": max(1, _coerce_int(entry.get("timeout"), 20)), + "search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)), + "base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"), + } + + def _resolve_settings( + self, + *, + filters: Optional[Dict[str, Any]] = None, + instance_name: Optional[str] = None, + require_explicit: bool = False, + ) -> Dict[str, Any]: + requested = self.requested_instance_name(filters, instance=instance_name) + resolved_name, conf = self.resolve_plugin_instance( + requested, + require_explicit=require_explicit or bool(requested), + ) + settings = self._settings_from_config(conf, instance_name=resolved_name) + if settings.get("instance") is None and requested: + settings["instance"] = requested + return settings def validate(self) -> bool: - return bool(self._host) + settings = self._resolve_settings() + return bool(settings.get("host")) def config_helper_text(self) -> str: return "Test the configured FTP/FTPS settings before searching or uploading." @@ -186,13 +208,14 @@ class FTP(Provider): if str(action_id or "").strip().lower() != "test_connection": return super().run_config_action(action_id, **_kwargs) - if not self._host: + settings = self._resolve_settings() + if not settings.get("host"): return {"ok": False, "message": "Set 'host' before testing the FTP connection."} ftp = None try: - ftp = self._connect() - active_path = self._base_path or "/" + ftp = self._connect(settings=settings) + active_path = str(settings.get("base_path") or "/") try: ftp.cwd(active_path) resolved_path = ftp.pwd() @@ -200,7 +223,7 @@ class FTP(Provider): resolved_path = active_path return { "ok": True, - "message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.", + "message": f"Connected to FTP {settings.get('host')}:{settings.get('port')} and reached {resolved_path}.", } except Exception as exc: return {"ok": False, "message": f"FTP connection failed: {exc}"} @@ -211,6 +234,10 @@ class FTP(Provider): text, inline = parse_inline_query_arguments(query) filters: Dict[str, Any] = {} + instance_name = str(inline.get("instance") or inline.get("store") or "").strip() + if instance_name: + filters["instance"] = instance_name + if inline.get("path"): filters["path"] = inline.get("path") if inline.get("depth"): @@ -221,17 +248,21 @@ class FTP(Provider): return text, filters def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: - active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) + settings = self._resolve_settings(filters=filters) + active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")) + instance_name = str(settings.get("instance") or "").strip() text = str(query or "").strip() if not text or text == "*": - return f"FTP: {active_path}" - return f"FTP: {text} @ {active_path}" + return f"FTP{f'[{instance_name}]' if instance_name else ''}: {active_path}" + return f"FTP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}" def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + settings = self._resolve_settings(filters=filters) return { "plugin": self.name, - "host": self._host, - "path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), + "instance": settings.get("instance"), + "host": settings.get("host"), + "path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")), "query": str(query or "").strip(), } @@ -244,15 +275,21 @@ class FTP(Provider): ) -> List[SearchResult]: _ = kwargs active_filters = dict(filters or {}) - start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) - search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) + settings = self._resolve_settings(filters=active_filters, require_explicit=True) + if not settings.get("host"): + requested = self.requested_instance_name(active_filters) + if requested: + raise RuntimeError(f"FTP instance '{requested}' is unavailable") + return [] + start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")) + search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth))) type_filter = str(active_filters.get("type") or "any").strip().lower() needle = str(query or "").strip() max_results = max(0, int(limit or 0)) if max_results <= 0: return [] - ftp = self._connect() + ftp = self._connect(settings=settings) try: return self._search_directory( ftp, @@ -261,6 +298,7 @@ class FTP(Provider): limit=max_results, search_depth=search_depth, type_filter=type_filter, + settings=settings, ) finally: self._close(ftp) @@ -278,19 +316,23 @@ class FTP(Provider): target_path = "" target_title = "" + instance_name = "" for item in selected_items or []: metadata = self._item_metadata(item) if not metadata.get("is_dir"): continue - target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path) + settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance"))) + target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/")) target_title = str(metadata.get("title") or metadata.get("name") or "").strip() + instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip() if target_path: break if not target_path: return False - ftp = self._connect() + settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name)) + ftp = self._connect(settings=settings) try: rows = self._search_directory( ftp, @@ -299,6 +341,7 @@ class FTP(Provider): limit=500, search_depth=0, type_filter="any", + settings=settings, ) finally: self._close(ftp) @@ -310,18 +353,23 @@ class FTP(Provider): return True title = target_title or target_path - table = Table(f"FTP: {title}")._perseverance(True) + table = Table(f"FTP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True) table.set_table("ftp") try: table.set_table_metadata({ "provider": "ftp", - "host": self._host, + "instance": instance_name or None, + "host": settings.get("host"), "path": target_path, "view": "directory", }) except Exception: pass - table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"]) + source_args = ["-plugin", "ftp"] + if instance_name: + source_args.extend(["-instance", instance_name]) + source_args.extend([f"path:{target_path}", "*"]) + table.set_source_command("search-file", source_args) payloads: List[Dict[str, Any]] = [] for row in rows: @@ -329,7 +377,7 @@ class FTP(Provider): payloads.append(row.to_dict()) try: - ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path}) + ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "instance": instance_name or None, "path": target_path}) ctx.set_current_stage_table(table) except Exception: pass @@ -342,6 +390,77 @@ class FTP(Provider): return True + def show_selection_details( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + source_command: str = "", + table_type: str = "", + table_metadata: Optional[Dict[str, Any]] = None, + **_kwargs: Any, + ) -> bool: + _ = table_type + item, _payload, _meta = self.resolve_selection_detail_subject( + selected_items, + stage_is_last=stage_is_last, + source_command=source_command, + require_media_kind="file", + ) + if item is None: + return False + + metadata = self._item_metadata(item) + if bool(metadata.get("is_dir")): + return False + + title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "FTP Item" + instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip() + ftp_url = str(metadata.get("ftp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip() + remote_path = str(metadata.get("ftp_path") or "").strip() + host = str(metadata.get("host") or "").strip() + modified = str(metadata.get("modified") or "").strip() + + try: + from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view + except Exception: + return super().show_selection_details( + selected_items, + ctx=ctx, + stage_is_last=stage_is_last, + source_command=source_command, + table_type=table_type, + table_metadata=table_metadata, + ) + + detail_metadata = prepare_detail_metadata( + item, + title=title, + store=instance_name or self.name, + path=ftp_url or remote_path or None, + tags=metadata.get("tag") or metadata.get("tags"), + extra_fields={ + "Plugin": self.name, + "Host": host or None, + "Instance": instance_name or None, + "Remote Path": remote_path or None, + "Directory": str(metadata.get("detail") or "").strip() or None, + "Modified": modified or None, + "Ftp Url": ftp_url or None, + }, + ) + + return render_selection_detail_view( + ctx=ctx, + item=item, + title=f"FTP Item: {title}", + metadata=detail_metadata, + table_name=self.name, + detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "FTP URL", "Plugin"], + value_case="preserve", + ) + def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: metadata = getattr(result, "full_metadata", None) if isinstance(metadata, dict) and metadata.get("is_dir"): @@ -349,10 +468,15 @@ class FTP(Provider): target = str(getattr(result, "path", "") or "").strip() if not target: return None - return self.download_url(target, output_dir, title=getattr(result, "title", None)) + instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else "" + return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None) def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: - settings = self._connection_settings_for_url(url) + parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {} + settings = self._connection_settings_for_url( + url, + instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None, + ) remote_path = settings["path"] if not remote_path or remote_path == "/": return None @@ -365,13 +489,7 @@ class FTP(Provider): destination_dir.mkdir(parents=True, exist_ok=True) destination = _unique_path(destination_dir / filename) - ftp = self._connect( - host=settings["host"], - port=settings["port"], - username=settings["username"], - password=settings["password"], - tls=settings["tls"], - ) + ftp = self._connect(settings=settings) try: with destination.open("wb") as handle: ftp.retrbinary(f"RETR {remote_path}", handle.write) @@ -404,7 +522,12 @@ class FTP(Provider): return None, None, None temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-")) - downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) + downloaded = self.download_url( + download_url, + temp_dir, + title=metadata.get("title"), + instance=metadata.get("instance"), + ) if downloaded is None: try: temp_dir.rmdir() @@ -424,35 +547,57 @@ class FTP(Provider): if not local_path.exists() or not local_path.is_file(): raise FileNotFoundError(f"File not found: {local_path}") - remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) + settings = self._resolve_settings( + instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None, + require_explicit=bool(kwargs.get("instance") or kwargs.get("store")), + ) + if not settings.get("host"): + requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip() + if requested: + raise RuntimeError(f"FTP instance '{requested}' is unavailable") + raise RuntimeError("No configured FTP instance is available") + + remote_dir = self._normalize_remote_path( + kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/", + default=str(settings.get("base_path") or "/"), + ) remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name remote_path = self._join_remote_path(remote_dir, remote_name) - ftp = self._connect() + ftp = self._connect(settings=settings) try: - self._ensure_directory(ftp, remote_dir) + self._ensure_directory(ftp, remote_dir, base_path=str(settings.get("base_path") or "/")) with local_path.open("rb") as handle: ftp.storbinary(f"STOR {remote_path}", handle) finally: self._close(ftp) - return self._build_url(remote_path) + return self._build_url(remote_path, settings=settings) def _connect( self, *, + settings: Optional[Dict[str, Any]] = None, host: Optional[str] = None, port: Optional[int] = None, username: Optional[str] = None, password: Optional[str] = None, tls: Optional[bool] = None, ) -> ftplib.FTP: - use_tls = self._tls if tls is None else bool(tls) + resolved = dict(settings or {}) + use_tls = bool(resolved.get("tls")) if tls is None else bool(tls) ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP() - ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout) - ftp.login(username or self._username, password or self._password) + ftp.connect( + host or str(resolved.get("host") or self._host), + int(port or resolved.get("port") or self._port), + timeout=max(1, int(resolved.get("timeout") or self._timeout)), + ) + ftp.login( + username or str(resolved.get("username") or self._username), + password or str(resolved.get("password") or self._password), + ) try: - ftp.set_pasv(self._passive) + ftp.set_pasv(bool(resolved.get("passive")) if "passive" in resolved else self._passive) except Exception: pass if use_tls and isinstance(ftp, ftplib.FTP_TLS): @@ -497,32 +642,39 @@ class FTP(Provider): self, remote_path: Any, *, + settings: Optional[Dict[str, Any]] = None, host: Optional[str] = None, port: Optional[int] = None, tls: Optional[bool] = None, ) -> str: + resolved = dict(settings or {}) path_text = self._normalize_remote_path(remote_path, default="/") - scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp" - host_text = str(host or self._host).strip() - port_value = int(port or self._port) + scheme = "ftps" if ((bool(resolved.get("tls")) if tls is None else bool(tls))) else "ftp" + host_text = str(host or resolved.get("host") or self._host).strip() + port_value = int(port or resolved.get("port") or self._port) port_suffix = f":{port_value}" if port_value and port_value != 21 else "" return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" - def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: + def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]: + settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name)) parsed = urlparse(str(url or "").strip()) scheme = (parsed.scheme or "ftp").strip().lower() - host = parsed.hostname or self._host - port = parsed.port or self._port - username = parsed.username or self._username - password = parsed.password or self._password - path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") + host = parsed.hostname or settings.get("host") or self._host + port = parsed.port or settings.get("port") or self._port + username = parsed.username or settings.get("username") or self._username + password = parsed.password or settings.get("password") or self._password + path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/")) return { + "instance": settings.get("instance"), "tls": scheme == "ftps", "host": host, "port": port, "username": username, "password": password, "path": path_text, + "passive": settings.get("passive", self._passive), + "timeout": settings.get("timeout", self._timeout), + "base_path": settings.get("base_path", self._base_path), } def _search_directory( @@ -534,21 +686,22 @@ class FTP(Provider): limit: int, search_depth: int, type_filter: str, + settings: Dict[str, Any], ) -> List[SearchResult]: results: List[SearchResult] = [] visited: set[str] = set() def walk(current_path: str, depth_left: int) -> None: - normalized = self._normalize_remote_path(current_path, default=self._base_path) + normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path)) if normalized in visited or len(results) >= limit: return visited.add(normalized) - for entry in self._list_directory(ftp, normalized): + for entry in self._list_directory(ftp, normalized, base_path=str(settings.get("base_path") or self._base_path)): if len(results) >= limit: return if self._matches_entry(entry, needle=needle, type_filter=type_filter): - results.append(self._build_result(entry)) + results.append(self._build_result(entry, settings=settings)) if entry.get("is_dir") and depth_left > 0: walk(str(entry.get("ftp_path") or normalized), depth_left - 1) @@ -578,16 +731,18 @@ class FTP(Provider): return False return True - def _build_result(self, entry: Dict[str, Any]) -> SearchResult: + def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult: ftp_path = str(entry.get("ftp_path") or "/") - ftp_url = self._build_url(ftp_path) + ftp_url = self._build_url(ftp_path, settings=settings) is_dir = bool(entry.get("is_dir")) size_value = entry.get("size") modified = str(entry.get("modified") or "") parent = posixpath.dirname(ftp_path.rstrip("/")) or "/" + instance_name = str(settings.get("instance") or "").strip() metadata = { "provider": "ftp", - "host": self._host, + "instance": instance_name or None, + "host": settings.get("host"), "ftp_path": ftp_path, "ftp_url": ftp_url, "selection_url": ftp_url, @@ -599,6 +754,13 @@ class FTP(Provider): if modified: metadata["modified"] = modified + selection_args = ["-url", ftp_url] + selection_action = ["download-file", "-plugin", "ftp"] + if instance_name: + selection_args = ["-instance", instance_name, *selection_args] + selection_action.extend(["-instance", instance_name]) + selection_action.extend(["-url", ftp_url]) + return SearchResult( table="ftp", title=str(entry.get("name") or ftp_path), @@ -615,13 +777,13 @@ class FTP(Provider): ("Size", "" if size_value is None else str(size_value)), ("Modified", modified), ], - selection_args=None if is_dir else ["-url", ftp_url], - selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url], + selection_args=None if is_dir else selection_args, + selection_action=None if is_dir else selection_action, full_metadata=metadata, ) - def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]: - normalized = self._normalize_remote_path(remote_path, default=self._base_path) + def _list_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> List[Dict[str, Any]]: + normalized = self._normalize_remote_path(remote_path, default=base_path) try: entries: List[Dict[str, Any]] = [] for name, facts in ftp.mlsd(normalized): @@ -716,8 +878,8 @@ class FTP(Provider): return _format_timestamp(parts[1]) return "" - def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None: - normalized = self._normalize_remote_path(remote_path, default=self._base_path) + def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> None: + normalized = self._normalize_remote_path(remote_path, default=base_path) if normalized == "/": return partial = "" @@ -763,11 +925,18 @@ class FTP(Provider): if path_text.startswith(("ftp://", "ftps://")): ftp_path = self._normalize_remote_path(path_text, default=self._base_path) if ftp_path: - metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path) + base_path = str(metadata.get("base_path") or self._base_path) + metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=base_path) metadata.setdefault("selection_path", metadata["ftp_path"]) if metadata.get("ftp_path") and not metadata.get("ftp_url"): - metadata["ftp_url"] = self._build_url(metadata["ftp_path"]) + metadata["ftp_url"] = self._build_url( + metadata["ftp_path"], + settings={ + "host": metadata.get("host") or self._host, + "instance": metadata.get("instance"), + }, + ) if metadata.get("ftp_url") and not metadata.get("selection_url"): metadata["selection_url"] = metadata["ftp_url"] diff --git a/plugins/libgen/__init__.py b/plugins/libgen/__init__.py index b61463a..0f4469f 100644 --- a/plugins/libgen/__init__.py +++ b/plugins/libgen/__init__.py @@ -286,7 +286,7 @@ def _enrich_book_tags_from_isbn(isbn: str, Priority: 1) OpenLibrary API-by-ISBN scrape (fast, structured) - 2) isbnsearch.org scrape via MetadataProvider + 2) isbnsearch.org scrape via metadata plugin """ isbn_clean = re.sub(r"[^0-9Xx]", "", str(isbn or "")).upper() @@ -381,12 +381,12 @@ def _enrich_book_tags_from_isbn(isbn: str, except Exception: pass - # 2) isbnsearch metadata provider fallback. + # 2) isbnsearch metadata plugin fallback. try: - from plugins.metadata_provider import get_metadata_provider + from plugins.metadata_provider import get_metadata_plugin - provider = get_metadata_provider("isbnsearch", - config or {}) + provider = get_metadata_plugin("isbnsearch", + config or {}) if provider is None: return [], "" items = provider.search(isbn_clean, limit=1) diff --git a/plugins/metadata_provider.py b/plugins/metadata_provider.py index 3b37ecc..e2d6bc7 100644 --- a/plugins/metadata_provider.py +++ b/plugins/metadata_provider.py @@ -1,7 +1,2031 @@ -"""Plugin-namespace import shim for metadata helper utilities. +from __future__ import annotations -The implementation currently lives in ``Provider.metadata_provider`` while the -legacy namespace is phased out. New imports should prefer ``plugins``. -""" +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Type, cast +import html as html_std +import re +import sys +import json +import subprocess -from Provider.metadata_provider import * # noqa: F401,F403 +from API.HTTP import HTTPClient +from API.requests_client import get_requests_session +from ProviderCore.base import SearchResult +try: + from plugins.tidal import Tidal +except ImportError: # pragma: no cover - optional + Tidal = None +from API.Tidal import ( + build_track_tags, + extract_artists, + stringify, +) +try: # Optional dependency for IMDb scraping + from imdbinfo.services import search_title # type: ignore +except ImportError: # pragma: no cover - optional + search_title = None # type: ignore[assignment] + +from SYS.logger import log, debug +from SYS.metadata import imdb_tag +from SYS.json_table import normalize_record + +try: # Optional dependency + import musicbrainzngs # type: ignore +except ImportError: # pragma: no cover - optional + musicbrainzngs = None + +try: # Optional dependency + import yt_dlp # type: ignore +except ImportError: # pragma: no cover - optional + yt_dlp = None + + +def _dedup_text_values(values: List[str]) -> List[str]: + out: List[str] = [] + seen: set[str] = set() + for value in values or []: + if value is None: + continue + text = str(value).strip() + if not text: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + out.append(text) + return out + + +def _filter_default_scraped_tags(tags: List[str]) -> List[str]: + blocked = {"title", "artist", "source"} + out: List[str] = [] + seen: set[str] = set() + for tag in tags or []: + text = str(tag or "").strip() + if not text: + continue + namespace = text.split(":", 1)[0].strip().lower() if ":" in text else "" + if namespace in blocked: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + out.append(text) + return out + + +class MetadataPlugin(ABC): + """Base class for metadata plugins (music, movies, books, etc.).""" + + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + self.config = config or {} + + @property + def name(self) -> str: + class_name = self.__class__.__name__ + if class_name.endswith("MetadataPlugin"): + return class_name[: -len("MetadataPlugin")].lower() + if class_name.endswith("Provider"): + return class_name[: -len("Provider")].lower() + return class_name.lower() + + @abstractmethod + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + """Return a list of candidate metadata records.""" + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + """Convert a result item into a list of tags.""" + tags: List[str] = [] + title = item.get("title") + artist = item.get("artist") + album = item.get("album") + year = item.get("year") + + if title: + tags.append(f"title:{title}") + if artist: + tags.append(f"artist:{artist}") + if album: + tags.append(f"album:{album}") + if year: + tags.append(f"year:{year}") + + tags.append(f"source:{self.name}") + return tags + + def search_tags(self, query: str, limit: int = 1) -> List[str]: + """Return tags for the best match from `search(query)`. + + Plugins can override this when tags should be extracted differently from + the default search->first-item->to_tags flow. + """ + + try: + items = self.search(query, limit=max(1, int(limit))) + except Exception: + return [] + if not items: + return [] + try: + return [str(t) for t in self.to_tags(items[0]) if t is not None] + except Exception: + return [] + + def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: + """Return plugin-specific identifier query text from parsed identifiers.""" + + _ = identifiers + return None + + def combined_query( + self, + *, + title_hint: Optional[str], + artist_hint: Optional[str], + ) -> Optional[str]: + """Return plugin-specific title+artist query text.""" + + _ = title_hint + _ = artist_hint + return None + + def extract_url_query(self, result: Any, get_field: Any) -> Optional[str]: + """Return plugin-specific URL query derived from a piped result.""" + + _ = result + _ = get_field + return None + + def emits_direct_tags(self) -> bool: + """True when the plugin should skip selection table and emit tags directly.""" + + return False + + def default_subject_scrape_priority(self) -> int: + """Priority used when `get-tag -scrape` is invoked without an explicit plugin.""" + + return 0 + + def url_scrape_priority(self, url: str) -> int: + """Priority for handling a raw URL passed to `get-tag -scrape `.""" + + _ = url + return 0 + + def resolve_subject_query( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> Optional[str]: + """Resolve a plugin-specific query from the current subject/result.""" + + _ = backend + _ = file_hash + return self.extract_url_query(result, get_field) + + def prefers_store_tag_overwrite(self) -> bool: + """Whether direct subject scrapes should replace the store tag set.""" + + return False + + def filter_tags_for_selection(self, tags: List[str]) -> List[str]: + """Filter scraped tags before presenting a selectable metadata row.""" + + return _filter_default_scraped_tags(tags) + + def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: + """Filter scraped tags before applying them to an existing store-backed item.""" + + return self.filter_tags_for_selection(tags) + + def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: + """Return a URL scrape payload for `get-tag -scrape ` when supported.""" + + items = self.search(url, limit=1) + if not items: + return None + item = items[0] if isinstance(items[0], dict) else {} + try: + tags = [str(t) for t in self.to_tags(item) if t is not None] + except Exception: + tags = [] + return { + "title": item.get("title"), + "tag": _dedup_text_values(tags), + "formats": [], + "playlist_items": [], + } + + +class ITunesMetadataPlugin(MetadataPlugin): + """Metadata plugin using the iTunes Search API.""" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + params = { + "term": query, + "media": "music", + "entity": "song", + "limit": limit + } + try: + resp = get_requests_session().get( + "https://itunes.apple.com/search", + params=params, + timeout=10 + ) + resp.raise_for_status() + results = resp.json().get("results", []) + except Exception as exc: + log(f"iTunes search failed: {exc}", file=sys.stderr) + return [] + + items: List[Dict[str, Any]] = [] + for r in results: + item = { + "title": r.get("trackName"), + "artist": r.get("artistName"), + "album": r.get("collectionName"), + "year": str(r.get("releaseDate", + ""))[:4], + "provider": self.name, + "raw": r, + } + items.append(item) + debug(f"iTunes returned {len(items)} items for '{query}'") + return items + + def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: + return identifiers.get("musicbrainz") or identifiers.get("musicbrainzalbum") + + def combined_query( + self, + *, + title_hint: Optional[str], + artist_hint: Optional[str], + ) -> Optional[str]: + title_text = str(title_hint or "").strip() + artist_text = str(artist_hint or "").strip() + if not title_text or not artist_text: + return None + return f"{title_text} {artist_text}" + + +class OpenLibraryMetadataPlugin(MetadataPlugin): + """Metadata plugin for OpenLibrary book metadata.""" + + @property + def name(self) -> str: # type: ignore[override] + return "openlibrary" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + query_clean = (query or "").strip() + if not query_clean: + return [] + + try: + # Prefer ISBN-specific search when the query looks like one + if query_clean.replace("-", + "").isdigit() and len(query_clean.replace("-", + "")) in ( + 10, + 13, + ): + q = f"isbn:{query_clean.replace('-', '')}" + else: + q = query_clean + + resp = get_requests_session().get( + "https://openlibrary.org/search.json", + params={ + "q": q, + "limit": limit + }, + timeout=10, + ) + resp.raise_for_status() + data = resp.json() + except Exception as exc: + log(f"OpenLibrary search failed: {exc}", file=sys.stderr) + return [] + + items: List[Dict[str, Any]] = [] + for doc in data.get("docs", [])[:limit]: + authors = doc.get("author_name") or [] + publisher = "" + publishers = doc.get("publisher") or [] + if isinstance(publishers, list) and publishers: + publisher = publishers[0] + + # Prefer 13-digit ISBN when available, otherwise 10-digit + isbn_list = doc.get("isbn") or [] + isbn_13 = next((i for i in isbn_list if len(str(i)) == 13), None) + isbn_10 = next((i for i in isbn_list if len(str(i)) == 10), None) + + # Derive OLID from key + olid = "" + key = doc.get("key", "") + if isinstance(key, str) and key: + olid = key.split("/")[-1] + + items.append( + { + "title": doc.get("title") or "", + "artist": ", ".join(authors) if authors else "", + "album": publisher, + "year": str(doc.get("first_publish_year") or ""), + "provider": self.name, + "authors": authors, + "publisher": publisher, + "identifiers": { + "isbn_13": isbn_13, + "isbn_10": isbn_10, + "openlibrary": olid, + "oclc": (doc.get("oclc_numbers") or [None])[0], + "lccn": (doc.get("lccn") or [None])[0], + }, + "description": None, + } + ) + + return items + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + tags: List[str] = [] + title = item.get("title") + authors = item.get("authors") or [] + publisher = item.get("publisher") + year = item.get("year") + description = item.get("description") or "" + + if title: + tags.append(f"title:{title}") + for author in authors: + if author: + tags.append(f"author:{author}") + if publisher: + tags.append(f"publisher:{publisher}") + if year: + tags.append(f"year:{year}") + if description: + tags.append(f"description:{description[:200]}") + + identifiers = item.get("identifiers") or {} + for key, value in identifiers.items(): + if value: + tags.append(f"{key}:{value}") + + tags.append(f"source:{self.name}") + return tags + + def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: + return ( + identifiers.get("isbn_13") + or identifiers.get("isbn_10") + or identifiers.get("isbn") + or identifiers.get("openlibrary") + ) + + +class GoogleBooksMetadataPlugin(MetadataPlugin): + """Metadata plugin for Google Books volumes API.""" + + @property + def name(self) -> str: # type: ignore[override] + return "googlebooks" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + query_clean = (query or "").strip() + if not query_clean: + return [] + + # Prefer ISBN queries when possible + if query_clean.replace("-", + "").isdigit() and len(query_clean.replace("-", + "")) in (10, + 13): + q = f"isbn:{query_clean.replace('-', '')}" + else: + q = query_clean + + try: + resp = get_requests_session().get( + "https://www.googleapis.com/books/v1/volumes", + params={ + "q": q, + "maxResults": limit + }, + timeout=10, + ) + resp.raise_for_status() + payload = resp.json() + except Exception as exc: + log(f"Google Books search failed: {exc}", file=sys.stderr) + return [] + + items: List[Dict[str, Any]] = [] + for volume in payload.get("items", [])[:limit]: + info = volume.get("volumeInfo") or {} + authors = info.get("authors") or [] + publisher = info.get("publisher", "") + published_date = info.get("publishedDate", "") + year = str(published_date)[:4] if published_date else "" + + identifiers_raw = info.get("industryIdentifiers") or [] + identifiers: Dict[str, + Optional[str]] = { + "googlebooks": volume.get("id") + } + for ident in identifiers_raw: + if not isinstance(ident, dict): + continue + ident_type = ident.get("type", "").lower() + ident_value = ident.get("identifier") + if not ident_value: + continue + if ident_type == "isbn_13": + identifiers.setdefault("isbn_13", ident_value) + elif ident_type == "isbn_10": + identifiers.setdefault("isbn_10", ident_value) + else: + identifiers.setdefault(ident_type, ident_value) + + items.append( + { + "title": info.get("title") or "", + "artist": ", ".join(authors) if authors else "", + "album": publisher, + "year": year, + "provider": self.name, + "authors": authors, + "publisher": publisher, + "identifiers": identifiers, + "description": info.get("description", + ""), + } + ) + + return items + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + tags: List[str] = [] + title = item.get("title") + authors = item.get("authors") or [] + publisher = item.get("publisher") + year = item.get("year") + description = item.get("description") or "" + + if title: + tags.append(f"title:{title}") + for author in authors: + if author: + tags.append(f"author:{author}") + if publisher: + tags.append(f"publisher:{publisher}") + if year: + tags.append(f"year:{year}") + if description: + tags.append(f"description:{description[:200]}") + + identifiers = item.get("identifiers") or {} + for key, value in identifiers.items(): + if value: + tags.append(f"{key}:{value}") + + tags.append(f"source:{self.name}") + return tags + + def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: + return ( + identifiers.get("isbn_13") + or identifiers.get("isbn_10") + or identifiers.get("isbn") + or identifiers.get("openlibrary") + ) + + +class ISBNsearchMetadataPlugin(MetadataPlugin): + """Metadata plugin that scrapes isbnsearch.org by ISBN. + + This is a best-effort HTML scrape. It expects the query to be an ISBN. + """ + + @property + def name(self) -> str: # type: ignore[override] + return "isbnsearch" + + @staticmethod + def _strip_html_to_text(raw: str) -> str: + s = html_std.unescape(str(raw or "")) + s = re.sub(r"(?i)", "\n", s) + s = re.sub(r"<[^>]+>", " ", s) + s = re.sub(r"\s+", " ", s) + return s.strip() + + @staticmethod + def _clean_isbn(query: str) -> str: + s = str(query or "").strip() + if not s: + return "" + s = s.replace("isbn:", "").replace("ISBN:", "") + s = re.sub(r"[^0-9Xx]", "", s).upper() + if len(s) in (10, 13): + return s + # Try to locate an ISBN-like token inside the query. + m = re.search(r"\b(?:97[89])?\d{9}[\dXx]\b", s) + return str(m.group(0)).upper() if m else "" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + _ = limit + isbn = self._clean_isbn(query) + if not isbn: + return [] + + url = f"https://isbnsearch.org/isbn/{isbn}" + try: + resp = get_requests_session().get(url, timeout=10) + resp.raise_for_status() + html = str(resp.text or "") + if not html: + return [] + except Exception as exc: + log(f"ISBNsearch scrape failed: {exc}", file=sys.stderr) + return [] + + title = "" + m_title = re.search(r"(?is)]*>(.*?)", html) + if m_title: + title = self._strip_html_to_text(m_title.group(1)) + + raw_fields: Dict[str, + str] = {} + strong_matches = list(re.finditer(r"(?is)]*>(.*?)", html)) + for idx, m in enumerate(strong_matches): + label_raw = self._strip_html_to_text(m.group(1)) + label = str(label_raw or "").strip() + if not label: + continue + if label.endswith(":"): + label = label[:-1].strip() + + chunk_start = m.end() + # Stop at next or end of document. + chunk_end = ( + strong_matches[idx + 1].start() if + (idx + 1) < len(strong_matches) else len(html) + ) + chunk = html[chunk_start:chunk_end] + # Prefer stopping within the same paragraph when possible. + m_end = re.search(r"(?is)(

|)", chunk) + if m_end: + chunk = chunk[:m_end.start()] + + val_text = self._strip_html_to_text(chunk) + if not val_text: + continue + raw_fields[label] = val_text + + def _get(*labels: str) -> str: + for lab in labels: + for k, v in raw_fields.items(): + if str(k).strip().lower() == str(lab).strip().lower(): + return str(v or "").strip() + return "" + + # Map common ISBNsearch labels. + author_text = _get("Author", "Authors", "Author(s)") + publisher = _get("Publisher") + published = _get("Published", "Publication Date", "Publish Date") + language = _get("Language") + pages = _get("Pages") + isbn_13 = _get("ISBN-13", "ISBN13") + isbn_10 = _get("ISBN-10", "ISBN10") + + year = "" + if published: + m_year = re.search(r"\b(\d{4})\b", published) + year = str(m_year.group(1)) if m_year else "" + + authors: List[str] = [] + if author_text: + # Split on common separators; keep multi-part names intact. + for part in re.split(r"\s*(?:,|;|\band\b|\&|\|)\s*", + author_text, + flags=re.IGNORECASE): + p = str(part or "").strip() + if p: + authors.append(p) + + # Prefer parsed title, but fall back to og:title if needed. + if not title: + m_og = re.search( + r"(?is)]*property=['\"]og:title['\"][^>]*content=['\"](.*?)['\"][^>]*>", + html, + ) + if m_og: + title = self._strip_html_to_text(m_og.group(1)) + + # Ensure ISBN tokens are normalized. + isbn_tokens: List[str] = [] + for token in [isbn_13, isbn_10, isbn]: + t = self._clean_isbn(token) + if t and t not in isbn_tokens: + isbn_tokens.append(t) + + item: Dict[str, + Any] = { + "title": title or "", + # Keep UI columns compatible with the generic metadata table. + "artist": ", ".join(authors) if authors else "", + "album": publisher or "", + "year": year or "", + "provider": self.name, + "authors": authors, + "publisher": publisher or "", + "language": language or "", + "pages": pages or "", + "identifiers": { + "isbn_13": + next((t for t in isbn_tokens if len(t) == 13), + None), + "isbn_10": + next((t for t in isbn_tokens if len(t) == 10), + None), + }, + "raw_fields": raw_fields, + } + + # Only return usable items. + if not item.get("title") and not any(item["identifiers"].values()): + return [] + + return [item] + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + tags: List[str] = [] + + title = str(item.get("title") or "").strip() + if title: + tags.append(f"title:{title}") + + authors = item.get("authors") or [] + if isinstance(authors, list): + for a in authors: + a = str(a or "").strip() + if a: + tags.append(f"author:{a}") + + publisher = str(item.get("publisher") or "").strip() + if publisher: + tags.append(f"publisher:{publisher}") + + year = str(item.get("year") or "").strip() + if year: + tags.append(f"year:{year}") + + language = str(item.get("language") or "").strip() + if language: + tags.append(f"language:{language}") + + identifiers = item.get("identifiers") or {} + if isinstance(identifiers, dict): + for key in ("isbn_13", "isbn_10"): + val = identifiers.get(key) + if val: + tags.append(f"isbn:{val}") + + tags.append(f"source:{self.name}") + + # Dedup case-insensitively, preserve order. + seen: set[str] = set() + out: List[str] = [] + for t in tags: + s = str(t or "").strip() + if not s: + continue + k = s.lower() + if k in seen: + continue + seen.add(k) + out.append(s) + return out + + +class MusicBrainzMetadataPlugin(MetadataPlugin): + """Metadata plugin for MusicBrainz recordings.""" + + @property + def name(self) -> str: # type: ignore[override] + return "musicbrainz" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + if not musicbrainzngs: + log( + "musicbrainzngs is not installed; skipping MusicBrainz scrape", + file=sys.stderr + ) + return [] + + q = (query or "").strip() + if not q: + return [] + + try: + # Ensure user agent is set (required by MusicBrainz) + musicbrainzngs.set_useragent("Medeia-Macina", "0.1") + except Exception: + pass + + try: + resp = musicbrainzngs.search_recordings(query=q, limit=limit) + recordings = resp.get("recording-list") or resp.get("recordings") or [] + except Exception as exc: + log(f"MusicBrainz search failed: {exc}", file=sys.stderr) + return [] + + items: List[Dict[str, Any]] = [] + for rec in recordings[:limit]: + if not isinstance(rec, dict): + continue + title = rec.get("title") or "" + + artist = "" + artist_credit = rec.get("artist-credit") or rec.get("artist_credit") + if isinstance(artist_credit, list) and artist_credit: + first = artist_credit[0] + if isinstance(first, dict): + artist = first.get("name") or first.get("artist", + {}).get("name", + "") + elif isinstance(first, str): + artist = first + + album = "" + release_list = rec.get("release-list") or rec.get("releases" + ) or rec.get("release") + if isinstance(release_list, list) and release_list: + first_rel = release_list[0] + if isinstance(first_rel, dict): + album = first_rel.get("title", "") or "" + release_date = first_rel.get("date") or "" + else: + album = str(first_rel) + release_date = "" + else: + release_date = rec.get("first-release-date") or "" + + year = str(release_date)[:4] if release_date else "" + mbid = rec.get("id") or "" + + items.append( + { + "title": title, + "artist": artist, + "album": album, + "year": year, + "provider": self.name, + "mbid": mbid, + "raw": rec, + } + ) + + return items + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + tags = super().to_tags(item) + mbid = item.get("mbid") + if mbid: + tags.append(f"musicbrainz:{mbid}") + return tags + + def combined_query( + self, + *, + title_hint: Optional[str], + artist_hint: Optional[str], + ) -> Optional[str]: + title_text = str(title_hint or "").strip() + artist_text = str(artist_hint or "").strip() + if not title_text or not artist_text: + return None + return f'recording:"{title_text}" AND artist:"{artist_text}"' + + +class ImdbMetadataPlugin(MetadataPlugin): + """Metadata plugin for IMDb titles (movies/series/episodes).""" + + @property + def name(self) -> str: # type: ignore[override] + return "imdb" + + @staticmethod + def _extract_imdb_id(text: str) -> str: + raw = str(text or "").strip() + if not raw: + return "" + + # Exact tt123 pattern + m = re.search(r"(tt\d+)", raw, re.IGNORECASE) + if m: + imdb_id = m.group(1).lower() + return imdb_id if imdb_id.startswith("tt") else f"tt{imdb_id}" + + # Bare numeric IDs (e.g., "0118883") + if raw.isdigit() and len(raw) >= 6: + return f"tt{raw}" + + # Last-resort: extract first digit run + m_digits = re.search(r"(\d{6,})", raw) + if m_digits: + return f"tt{m_digits.group(1)}" + + return "" + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + q = (query or "").strip() + if not q: + return [] + + imdb_id = self._extract_imdb_id(q) + if imdb_id: + try: + data = imdb_tag(imdb_id) + raw_tags = data.get("tag") if isinstance(data, dict) else [] + title = None + year = None + if isinstance(raw_tags, list): + for tag in raw_tags: + if not isinstance(tag, str): + continue + if tag.startswith("title:"): + title = tag.split(":", 1)[1] + elif tag.startswith("year:"): + year = tag.split(":", 1)[1] + return [ + { + "title": title or imdb_id, + "artist": "", + "album": "", + "year": str(year or ""), + "provider": self.name, + "imdb_id": imdb_id, + "raw": data, + } + ] + except Exception as exc: + log(f"IMDb lookup failed: {exc}", file=sys.stderr) + return [] + + if search_title is None: + log("imdbinfo is not installed; skipping IMDb scrape", file=sys.stderr) + return [] + + try: + search_result = search_title(q) + titles = getattr(search_result, "titles", None) or [] + except Exception as exc: + log(f"IMDb search failed: {exc}", file=sys.stderr) + return [] + + items: List[Dict[str, Any]] = [] + for entry in titles[:limit]: + imdb_id = self._extract_imdb_id( + getattr(entry, "imdb_id", None) + or getattr(entry, "imdbId", None) + or getattr(entry, "id", None) + ) + title = getattr(entry, "title", "") or getattr(entry, "title_localized", "") + year = str(getattr(entry, "year", "") or "")[:4] + kind = getattr(entry, "kind", "") or "" + rating = getattr(entry, "rating", None) + items.append( + { + "title": title, + "artist": "", + "album": kind, + "year": year, + "provider": self.name, + "imdb_id": imdb_id, + "kind": kind, + "rating": rating, + "raw": entry, + } + ) + return items + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + imdb_id = self._extract_imdb_id( + item.get("imdb_id") or item.get("id") or item.get("imdb") or "" + ) + try: + if imdb_id: + data = imdb_tag(imdb_id) + raw_tags = data.get("tag") if isinstance(data, dict) else [] + tags = [t for t in raw_tags if isinstance(t, str)] + if tags: + return tags + except Exception as exc: + log(f"IMDb tag extraction failed: {exc}", file=sys.stderr) + + tags = super().to_tags(item) + if imdb_id: + tags.append(f"imdb:{imdb_id}") + seen: set[str] = set() + deduped: List[str] = [] + for t in tags: + s = str(t or "").strip() + if not s: + continue + k = s.lower() + if k in seen: + continue + seen.add(k) + deduped.append(s) + return deduped + + def identifier_query(self, identifiers: Dict[str, Any]) -> Optional[str]: + return identifiers.get("imdb") + + +class YtdlpMetadataPlugin(MetadataPlugin): + """Metadata plugin that extracts tags from a supported URL using yt-dlp. + + This does NOT download media; it only probes metadata. + """ + + @property + def name(self) -> str: # type: ignore[override] + return "ytdlp" + + def _extract_info(self, url: str) -> Optional[Dict[str, Any]]: + url = (url or "").strip() + if not url: + return None + + # Prefer Python module when available. + if yt_dlp is not None: + try: + opts: Any = { + "quiet": True, + "no_warnings": True, + "skip_download": True, + "noprogress": True, + "socket_timeout": 15, + "retries": 1, + "playlist_items": "1-10", + } + with yt_dlp.YoutubeDL(opts) as ydl: # type: ignore[attr-defined] + info = ydl.extract_info(url, download=False) + return cast(Dict[str, Any], info) if isinstance(info, dict) else None + except Exception: + pass + + # Fallback to CLI. + try: + cmd = [ + "yt-dlp", + "-J", + "--no-warnings", + "--skip-download", + "--playlist-items", + "1-10", + url, + ] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if proc.returncode != 0: + return None + payload = (proc.stdout or "").strip() + if not payload: + return None + data = json.loads(payload) + return data if isinstance(data, dict) else None + except Exception: + return None + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + url = (query or "").strip() + if not url.startswith(("http://", "https://")): + return [] + + info = self._extract_info(url) + if not isinstance(info, dict): + return [] + + upload_date = str(info.get("upload_date") or "") + release_date = str(info.get("release_date") or "") + year = (release_date + or upload_date)[:4] if (release_date or upload_date) else "" + + # Provide basic columns for the standard metadata selection table. + # NOTE: This is best-effort; many extractors don't provide artist/album. + artist = info.get("artist") or info.get("uploader") or info.get("channel") or "" + album = info.get("album") or info.get("playlist_title") or "" + title = info.get("title") or "" + + return [ + { + "title": title, + "artist": str(artist or ""), + "album": str(album or ""), + "year": str(year or ""), + "provider": self.name, + "url": url, + "raw": info, + } + ] + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + raw = item.get("raw") + if not isinstance(raw, dict): + return super().to_tags(item) + + tags: List[str] = [] + try: + from SYS.yt_metadata import extract_ytdlp_tags + except Exception: + extract_ytdlp_tags = None # type: ignore[assignment] + + if extract_ytdlp_tags: + try: + tags.extend(extract_ytdlp_tags(raw)) + except Exception: + pass + + # Subtitle availability tags + def _langs(value: Any) -> List[str]: + if not isinstance(value, dict): + return [] + out: List[str] = [] + for k in value.keys(): + if isinstance(k, str) and k.strip(): + out.append(k.strip().lower()) + return sorted(set(out)) + + # If this is a playlist container, subtitle/captions are usually per-entry. + info_for_subs: Dict[str, Any] = raw + entries = raw.get("entries") + if isinstance(entries, list) and entries: + first = entries[0] + if isinstance(first, dict): + info_for_subs = first + + for lang in _langs(info_for_subs.get("subtitles")): + tags.append(f"subs:{lang}") + for lang in _langs(info_for_subs.get("automatic_captions")): + tags.append(f"subs_auto:{lang}") + + # Always include source tag for parity with other providers. + tags.append(f"source:{self.name}") + + # Dedup case-insensitively, preserve order. + seen = set() + out: List[str] = [] + for t in tags: + if not isinstance(t, str): + continue + s = t.strip() + if not s: + continue + k = s.lower() + if k in seen: + continue + seen.add(k) + out.append(s) + return out + + def extract_url_query(self, result: Any, get_field: Any) -> Optional[str]: + raw_url = ( + get_field(result, "url", None) + or get_field(result, "source_url", None) + or get_field(result, "target", None) + ) + if isinstance(raw_url, list) and raw_url: + raw_url = raw_url[0] + if isinstance(raw_url, str): + text = raw_url.strip() + if text.startswith(("http://", "https://")): + return text + return None + + def emits_direct_tags(self) -> bool: + return True + + def default_subject_scrape_priority(self) -> int: + return 100 + + def url_scrape_priority(self, url: str) -> int: + text = str(url or "").strip() + if not text.startswith(("http://", "https://")): + return 0 + return 100 + + def prefers_store_tag_overwrite(self) -> bool: + return True + + def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]: + return _dedup_text_values(tags) + + def _resolve_candidate_urls_for_subject( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> List[str]: + try: + from SYS.metadata import normalize_urls + except Exception: + normalize_urls = None # type: ignore[assignment] + + urls: List[str] = [] + + if backend is not None and file_hash: + try: + backend_urls = backend.get_url(file_hash, config=self.config) + if backend_urls: + if normalize_urls: + urls.extend(normalize_urls(backend_urls)) + else: + urls.extend( + [str(u).strip() for u in backend_urls if isinstance(u, str) and str(u).strip()] + ) + except Exception: + pass + + try: + meta = backend.get_metadata(file_hash, config=self.config) + if isinstance(meta, dict) and meta.get("url"): + raw = meta.get("url") + if normalize_urls: + urls.extend(normalize_urls(raw)) + elif isinstance(raw, list): + urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) + elif isinstance(raw, str) and raw.strip(): + urls.append(raw.strip()) + except Exception: + pass + + for key in ("url", "webpage_url", "source_url", "target"): + val = get_field(result, key, None) + if not val: + continue + if normalize_urls: + urls.extend(normalize_urls(val)) + continue + if isinstance(val, str) and val.strip(): + urls.append(val.strip()) + elif isinstance(val, list): + urls.extend([str(u).strip() for u in val if isinstance(u, str) and str(u).strip()]) + + meta_field = get_field(result, "metadata", None) + if isinstance(meta_field, dict) and meta_field.get("url"): + raw = meta_field.get("url") + if normalize_urls: + urls.extend(normalize_urls(raw)) + elif isinstance(raw, list): + urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()]) + elif isinstance(raw, str) and raw.strip(): + urls.append(raw.strip()) + + return _dedup_text_values(urls) + + def _pick_supported_subject_url(self, urls: List[str]) -> Optional[str]: + if not urls: + return None + + def _is_hydrus_file_url(u: str) -> bool: + text = str(u or "").strip().lower() + return bool(text and "/get_files/file" in text and "hash=" in text) + + candidates = [] + for url in urls: + text = str(url or "").strip() + if not text.startswith(("http://", "https://")): + continue + if _is_hydrus_file_url(text): + continue + candidates.append(text) + if not candidates: + return None + + try: + from tool.ytdlp import is_url_supported_by_ytdlp + + for text in candidates: + try: + if is_url_supported_by_ytdlp(text): + return text + except Exception: + continue + except Exception: + pass + + return candidates[0] if candidates else None + + def resolve_subject_query( + self, + result: Any, + get_field: Any, + *, + backend: Any = None, + file_hash: Optional[str] = None, + ) -> Optional[str]: + candidate_urls = self._resolve_candidate_urls_for_subject( + result, + get_field, + backend=backend, + file_hash=file_hash, + ) + return self._pick_supported_subject_url(candidate_urls) + + @staticmethod + def _extract_url_formats(formats: Any) -> List[tuple[str, str]]: + if not isinstance(formats, list): + return [] + + video_formats: Dict[str, Dict[str, Any]] = {} + audio_formats: Dict[str, Dict[str, Any]] = {} + + for fmt in formats: + if not isinstance(fmt, dict): + continue + vcodec = fmt.get("vcodec", "none") + acodec = fmt.get("acodec", "none") + height = fmt.get("height") + ext = fmt.get("ext", "unknown") + format_id = fmt.get("format_id", "") + tbr = fmt.get("tbr", 0) + abr = fmt.get("abr", 0) + + if vcodec and vcodec != "none" and height: + if int(height) < 480: + continue + res_key = f"{int(height)}p" + if res_key not in video_formats or tbr > video_formats[res_key].get("tbr", 0): + video_formats[res_key] = { + "label": f"{int(height)}p ({ext})", + "format_id": str(format_id), + "tbr": tbr, + } + elif acodec and acodec != "none" and (not vcodec or vcodec == "none"): + audio_key = f"audio_{abr}" + if audio_key not in audio_formats or abr > audio_formats[audio_key].get("abr", 0): + audio_formats[audio_key] = { + "label": f"audio ({ext})", + "format_id": str(format_id), + "abr": abr, + } + + result: List[tuple[str, str]] = [] + for res in sorted(video_formats.keys(), key=lambda value: int(value.replace("p", "")), reverse=True): + fmt = video_formats[res] + result.append((str(fmt.get("label") or res), str(fmt.get("format_id") or ""))) + if audio_formats: + best_audio_key = max(audio_formats.keys(), key=lambda key: float(audio_formats[key].get("abr", 0) or 0)) + fmt = audio_formats[best_audio_key] + result.append((str(fmt.get("label") or "audio"), str(fmt.get("format_id") or ""))) + return [entry for entry in result if entry[1]] + + @staticmethod + def _build_playlist_items(raw: Dict[str, Any]) -> List[Dict[str, Any]]: + entries = raw.get("entries") + if not isinstance(entries, list): + return [] + + playlist_items: List[Dict[str, Any]] = [] + for idx, entry in enumerate(entries, 1): + if not isinstance(entry, dict): + continue + playlist_items.append( + { + "index": idx, + "id": entry.get("id", f"track_{idx}"), + "title": entry.get("title", entry.get("id", f"Track {idx}")), + "duration": entry.get("duration", 0), + "url": entry.get("url") or entry.get("webpage_url", ""), + } + ) + return playlist_items + + def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]: + info = self._extract_info(url) + if not isinstance(info, dict): + return None + + item = { + "title": info.get("title") or "", + "artist": str(info.get("artist") or info.get("uploader") or info.get("channel") or ""), + "album": str(info.get("album") or info.get("playlist_title") or ""), + "year": str((str(info.get("release_date") or "") or str(info.get("upload_date") or ""))[:4]), + "provider": self.name, + "url": str(url or "").strip(), + "raw": info, + } + tags = _dedup_text_values([str(tag) for tag in self.to_tags(item) if tag is not None]) + return { + "title": item.get("title") or None, + "tag": tags, + "formats": self._extract_url_formats(info.get("formats", [])), + "playlist_items": self._build_playlist_items(info), + } + + +def _coerce_archive_field_list(value: Any) -> List[str]: + """Coerce an Archive.org metadata field to a list of strings.""" + + if value is None: + return [] + if isinstance(value, list): + out: List[str] = [] + for v in value: + try: + s = str(v).strip() + except Exception: + continue + if s: + out.append(s) + return out + if isinstance(value, (tuple, set)): + out = [] + for v in value: + try: + s = str(v).strip() + except Exception: + continue + if s: + out.append(s) + return out + try: + s = str(value).strip() + except Exception: + return [] + return [s] if s else [] + + +def archive_item_metadata_to_tags(archive_id: str, + item_metadata: Dict[str, Any]) -> List[str]: + """Coerce Archive.org metadata into a stable set of bibliographic tags.""" + + archive_id_clean = str(archive_id or "").strip() + meta = item_metadata if isinstance(item_metadata, dict) else {} + + tags: List[str] = [] + seen: set[str] = set() + + def _add(tag: str) -> None: + try: + t = str(tag).strip() + except Exception: + return + if not t: + return + if t.lower() in seen: + return + seen.add(t.lower()) + tags.append(t) + + if archive_id_clean: + _add(f"internet_archive:{archive_id_clean}") + + for title in _coerce_archive_field_list(meta.get("title"))[:1]: + _add(f"title:{title}") + + creators: List[str] = [] + creators.extend(_coerce_archive_field_list(meta.get("creator"))) + creators.extend(_coerce_archive_field_list(meta.get("author"))) + for creator in creators[:3]: + _add(f"author:{creator}") + + for publisher in _coerce_archive_field_list(meta.get("publisher"))[:3]: + _add(f"publisher:{publisher}") + + for date_val in _coerce_archive_field_list(meta.get("date"))[:1]: + _add(f"publish_date:{date_val}") + for year_val in _coerce_archive_field_list(meta.get("year"))[:1]: + _add(f"publish_date:{year_val}") + + for lang in _coerce_archive_field_list(meta.get("language"))[:3]: + _add(f"language:{lang}") + + for subj in _coerce_archive_field_list(meta.get("subject"))[:15]: + if len(subj) > 200: + subj = subj[:200] + _add(subj) + + def _clean_isbn(raw: str) -> str: + return str(raw or "").replace("-", "").strip() + + for isbn in _coerce_archive_field_list(meta.get("isbn"))[:10]: + isbn_clean = _clean_isbn(isbn) + if isbn_clean: + _add(f"isbn:{isbn_clean}") + + identifiers: List[str] = [] + identifiers.extend(_coerce_archive_field_list(meta.get("identifier"))) + identifiers.extend(_coerce_archive_field_list(meta.get("external-identifier"))) + added_other = 0 + for ident in identifiers: + ident_s = str(ident or "").strip() + if not ident_s: + continue + low = ident_s.lower() + + if low.startswith("urn:isbn:"): + val = _clean_isbn(ident_s.split(":", 2)[-1]) + if val: + _add(f"isbn:{val}") + continue + if low.startswith("isbn:"): + val = _clean_isbn(ident_s.split(":", 1)[-1]) + if val: + _add(f"isbn:{val}") + continue + if low.startswith("urn:oclc:"): + val = ident_s.split(":", 2)[-1].strip() + if val: + _add(f"oclc:{val}") + continue + if low.startswith("oclc:"): + val = ident_s.split(":", 1)[-1].strip() + if val: + _add(f"oclc:{val}") + continue + if low.startswith("urn:lccn:"): + val = ident_s.split(":", 2)[-1].strip() + if val: + _add(f"lccn:{val}") + continue + if low.startswith("lccn:"): + val = ident_s.split(":", 1)[-1].strip() + if val: + _add(f"lccn:{val}") + continue + if low.startswith("doi:"): + val = ident_s.split(":", 1)[-1].strip() + if val: + _add(f"doi:{val}") + continue + + if archive_id_clean and low == archive_id_clean.lower(): + continue + if added_other >= 5: + continue + if len(ident_s) > 200: + ident_s = ident_s[:200] + _add(f"identifier:{ident_s}") + added_other += 1 + + return tags + + +def fetch_archive_item_metadata(archive_id: str, + *, + timeout: int = 8) -> Dict[str, Any]: + ident = str(archive_id or "").strip() + if not ident: + return {} + resp = get_requests_session().get( + f"https://archive.org/metadata/{ident}", + timeout=int(timeout), + ) + resp.raise_for_status() + data = resp.json() if resp is not None else {} + if not isinstance(data, dict): + return {} + meta = data.get("metadata") + return meta if isinstance(meta, dict) else {} + + +def scrape_isbn_metadata(isbn: str) -> List[str]: + """Scrape metadata tags for an ISBN using OpenLibrary's books API.""" + + new_tags: List[str] = [] + + isbn_clean = str(isbn or "").replace("isbn:", "").replace("-", "").strip() + if not isbn_clean: + return [] + + url = f"https://openlibrary.org/api/books?bibkeys=ISBN:{isbn_clean}&jscmd=data&format=json" + try: + with HTTPClient() as client: + response = client.get(url) + response.raise_for_status() + data = json.loads(response.content.decode("utf-8")) + except Exception as exc: + log(f"Failed to fetch ISBN metadata: {exc}", file=sys.stderr) + return [] + + if not data: + log(f"No ISBN metadata found for: {isbn}") + return [] + + book_data = next(iter(data.values()), None) + if not isinstance(book_data, dict): + return [] + + if "title" in book_data: + new_tags.append(f"title:{book_data['title']}") + + authors = book_data.get("authors") + if isinstance(authors, list): + for author in authors[:3]: + if isinstance(author, dict) and author.get("name"): + new_tags.append(f"author:{author['name']}") + + if book_data.get("publish_date"): + new_tags.append(f"publish_date:{book_data['publish_date']}") + + publishers = book_data.get("publishers") + if isinstance(publishers, list) and publishers: + pub = publishers[0] + if isinstance(pub, dict) and pub.get("name"): + new_tags.append(f"publisher:{pub['name']}") + + if "description" in book_data: + desc = book_data.get("description") + if isinstance(desc, dict) and "value" in desc: + desc = desc.get("value") + if desc: + desc_str = str(desc).strip() + if desc_str: + new_tags.append(f"description:{desc_str[:200]}") + + page_count = book_data.get("number_of_pages") + if isinstance(page_count, int) and page_count > 0: + new_tags.append(f"pages:{page_count}") + + identifiers = book_data.get("identifiers") + if isinstance(identifiers, dict): + + def _first(value: Any) -> Any: + if isinstance(value, list) and value: + return value[0] + return value + + for key, ns in ( + ("openlibrary", "openlibrary"), + ("lccn", "lccn"), + ("oclc", "oclc"), + ("goodreads", "goodreads"), + ("librarything", "librarything"), + ("doi", "doi"), + ("internet_archive", "internet_archive"), + ): + val = _first(identifiers.get(key)) + if val: + new_tags.append(f"{ns}:{val}") + + debug(f"Found {len(new_tags)} tag(s) from ISBN lookup") + return new_tags + + +def scrape_openlibrary_metadata(olid: str) -> List[str]: + """Scrape metadata tags for an OpenLibrary ID using the edition JSON endpoint.""" + + new_tags: List[str] = [] + + olid_text = str(olid or "").strip() + if not olid_text: + return [] + + olid_norm = olid_text + try: + if not olid_norm.startswith("OL"): + olid_norm = f"OL{olid_norm}" + if not olid_norm.endswith("M"): + olid_norm = f"{olid_norm}M" + except Exception: + olid_norm = olid_text + + new_tags.append(f"openlibrary:{olid_norm}") + + olid_clean = olid_text.replace("OL", "").replace("M", "") + if not olid_clean.isdigit(): + olid_clean = olid_text + + if not olid_text.startswith("OL"): + url = f"https://openlibrary.org/books/OL{olid_clean}M.json" + else: + url = f"https://openlibrary.org/books/{olid_text}.json" + + try: + with HTTPClient() as client: + response = client.get(url) + response.raise_for_status() + data = json.loads(response.content.decode("utf-8")) + except Exception as exc: + log(f"Failed to fetch OpenLibrary metadata: {exc}", file=sys.stderr) + return [] + + if not isinstance(data, dict) or not data: + log(f"No OpenLibrary metadata found for: {olid_text}") + return [] + + if "title" in data: + new_tags.append(f"title:{data['title']}") + + authors = data.get("authors") + if isinstance(authors, list): + for author in authors[:3]: + if isinstance(author, dict) and author.get("name"): + new_tags.append(f"author:{author['name']}") + continue + + author_key = None + if isinstance(author, dict): + if isinstance(author.get("author"), dict): + author_key = author.get("author", {}).get("key") + if not author_key: + author_key = author.get("key") + + if isinstance(author_key, str) and author_key.startswith("/"): + try: + author_url = f"https://openlibrary.org{author_key}.json" + with HTTPClient(timeout=10) as client: + author_resp = client.get(author_url) + author_resp.raise_for_status() + author_data = json.loads(author_resp.content.decode("utf-8")) + if isinstance(author_data, dict) and author_data.get("name"): + new_tags.append(f"author:{author_data['name']}") + continue + except Exception: + pass + + if isinstance(author, str) and author: + new_tags.append(f"author:{author}") + + if data.get("publish_date"): + new_tags.append(f"publish_date:{data['publish_date']}") + + publishers = data.get("publishers") + if isinstance(publishers, list) and publishers: + pub = publishers[0] + if isinstance(pub, dict) and pub.get("name"): + new_tags.append(f"publisher:{pub['name']}") + elif isinstance(pub, str) and pub: + new_tags.append(f"publisher:{pub}") + + if "description" in data: + desc = data.get("description") + if isinstance(desc, dict) and "value" in desc: + desc = desc.get("value") + if desc: + desc_str = str(desc).strip() + if desc_str: + new_tags.append(f"description:{desc_str[:200]}") + + page_count = data.get("number_of_pages") + if isinstance(page_count, int) and page_count > 0: + new_tags.append(f"pages:{page_count}") + + subjects = data.get("subjects") + if isinstance(subjects, list): + for subject in subjects[:10]: + if isinstance(subject, str): + subject_clean = subject.strip() + if subject_clean and subject_clean not in new_tags: + new_tags.append(subject_clean) + + identifiers = data.get("identifiers") + if isinstance(identifiers, dict): + + def _first(value: Any) -> Any: + if isinstance(value, list) and value: + return value[0] + return value + + for key, ns in ( + ("isbn_10", "isbn_10"), + ("isbn_13", "isbn_13"), + ("lccn", "lccn"), + ("oclc_numbers", "oclc"), + ("goodreads", "goodreads"), + ("internet_archive", "internet_archive"), + ): + val = _first(identifiers.get(key)) + if val: + new_tags.append(f"{ns}:{val}") + + ocaid = data.get("ocaid") + if isinstance(ocaid, str) and ocaid.strip(): + new_tags.append(f"internet_archive:{ocaid.strip()}") + + debug(f"Found {len(new_tags)} tag(s) from OpenLibrary lookup") + return new_tags + + +SAMPLE_ITEMS: List[Dict[str, Any]] = [ + { + "title": "Sample OpenLibrary book", + "path": "https://openlibrary.org/books/OL123M", + "openlibrary_id": "OL123M", + "archive_id": "samplearchive123", + "availability": "borrow", + "availability_reason": "sample", + "direct_url": "https://archive.org/download/sample.pdf", + "author_name": ["OpenLibrary Demo"], + "first_publish_year": 2023, + "ia": ["samplearchive123"], + }, +] + + +try: + from typing import Iterable + + from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column + from SYS.result_table_adapters import register_plugin + + def _ensure_search_result(item: Any) -> SearchResult: + if isinstance(item, SearchResult): + return item + if isinstance(item, dict): + data = dict(item) + title = str(data.get("title") or data.get("name") or "OpenLibrary") + path = str(data.get("path") or data.get("url") or "") + detail = str(data.get("detail") or "") + annotations = list(data.get("annotations") or []) + media_kind = str(data.get("media_kind") or "book") + return SearchResult( + table="openlibrary", + title=title, + path=path, + detail=detail, + annotations=annotations, + media_kind=media_kind, + columns=data.get("columns") or [], + full_metadata={**data, "raw": dict(item)}, + ) + return SearchResult( + table="openlibrary", + title=str(item or "OpenLibrary"), + path="", + detail="", + annotations=[], + media_kind="book", + full_metadata={"raw": {}}, + ) + + def _adapter(items: Iterable[Any]) -> Iterable[ResultModel]: + for item in items: + sr = _ensure_search_result(item) + metadata = dict(getattr(sr, "full_metadata", {}) or {}) + raw = metadata.get("raw") + if isinstance(raw, dict): + normalized = normalize_record(raw) + for key, val in normalized.items(): + metadata.setdefault(key, val) + + def _make_url() -> str: + candidate = ( + metadata.get("selection_url") or + metadata.get("direct_url") or + metadata.get("url") or + metadata.get("path") or + sr.path or + "" + ) + return str(candidate or "").strip() + + selection_url = _make_url() + if selection_url: + metadata["selection_url"] = selection_url + authors_value = metadata.get("authors_display") or metadata.get("authors") or metadata.get("author_name") or "" + if isinstance(authors_value, list): + authors_value = ", ".join(str(v) for v in authors_value if v) + authors_text = str(authors_value or "").strip() + if authors_text: + metadata["authors_display"] = authors_text + year_value = metadata.get("year") or metadata.get("first_publish_year") + if year_value and not isinstance(year_value, str): + year_value = str(year_value) + if year_value: + metadata["year"] = str(year_value) + metadata.setdefault("openlibrary_id", metadata.get("openlibrary_id") or metadata.get("olid")) + metadata.setdefault("source", metadata.get("source") or "openlibrary") + yield ResultModel( + title=str(sr.title or metadata.get("title") or selection_url or "OpenLibrary"), + path=selection_url or None, + metadata=metadata, + source="openlibrary", + ) + + def _columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]: + cols: List[ColumnSpec] = [title_column()] + def _has(key: str) -> bool: + return any((row.metadata or {}).get(key) for row in rows) + + if _has("authors_display"): + cols.append( + ColumnSpec( + "authors_display", + "Author", + lambda r: (r.metadata or {}).get("authors_display") or "", + ) + ) + if _has("year"): + cols.append(metadata_column("year", "Year")) + if _has("availability"): + cols.append(metadata_column("availability", "Avail")) + if _has("archive_id"): + cols.append(metadata_column("archive_id", "Archive ID")) + if _has("openlibrary_id"): + cols.append(metadata_column("openlibrary_id", "OLID")) + return cols + + def _selection_fn(row: ResultModel) -> List[str]: + metadata = row.metadata or {} + url = str(metadata.get("selection_url") or row.path or "").strip() + if url: + return ["-url", url] + return ["-title", row.title or ""] + + register_plugin( + "openlibrary", + _adapter, + columns=_columns_factory, + selection_fn=_selection_fn, + metadata={"description": "OpenLibrary search provider (JSON result table template)"}, + ) +except Exception: + pass + + +# Registry --------------------------------------------------------------- + +class TidalMetadataPlugin(MetadataPlugin): + """Metadata plugin that reuses the Tidal search plugin for tidal info.""" + + @property + def name(self) -> str: # type: ignore[override] + return "tidal" + + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + if Tidal is None: + raise RuntimeError("Tidal provider unavailable for tidal metadata") + super().__init__(config) + self._provider = Tidal(self.config) + + def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + normalized = str(query or "").strip() + if not normalized: + return [] + + try: + results = self._provider.search(normalized, limit=limit) + except Exception as exc: + debug(f"[tidal-meta] search failed for '{normalized}': {exc}") + return [] + + items: List[Dict[str, Any]] = [] + for result in results: + metadata = getattr(result, "full_metadata", {}) or {} + if not isinstance(metadata, dict): + metadata = {} + + title = stringify(metadata.get("title") or result.title) + if not title: + continue + + artists = extract_artists(metadata) + artist_display = ", ".join(artists) if artists else stringify(metadata.get("artist")) + + album_obj = metadata.get("album") + album = "" + if isinstance(album_obj, dict): + album = stringify(album_obj.get("title")) + else: + album = stringify(metadata.get("album")) + + year = stringify(metadata.get("releaseDate") or metadata.get("year") or metadata.get("date")) + + track_id = self._provider._parse_track_id(metadata.get("trackId") or metadata.get("id")) + lyrics_data = None + if track_id is not None: + try: + lyrics_data = self._provider._fetch_track_lyrics(track_id) + except Exception as exc: + debug(f"[tidal-meta] lyrics lookup failed for {track_id}: {exc}") + + lyrics = None + if isinstance(lyrics_data, dict): + lyrics = stringify(lyrics_data.get("lyrics") or lyrics_data.get("text")) + subtitles = stringify(lyrics_data.get("subtitles")) + if subtitles: + metadata.setdefault("_tidal_lyrics", {})["subtitles"] = subtitles + + tags = sorted(build_track_tags(metadata)) + items.append({ + "title": title, + "artist": artist_display, + "album": album, + "year": year, + "lyrics": lyrics, + "tags": tags, + "provider": self.name, + "path": getattr(result, "path", ""), + "track_id": track_id, + "full_metadata": metadata, + }) + return items + + def to_tags(self, item: Dict[str, Any]) -> List[str]: + tags: List[str] = [] + for value in item.get("tags", []): + value_text = stringify(value) + if value_text: + normalized = value_text.lower() + if normalized in {"tidal", "lossless"}: + continue + if normalized.startswith("quality:lossless"): + continue + tags.append(value_text) + return tags + +_METADATA_PLUGINS: Dict[str, + Type[MetadataPlugin]] = { + "itunes": ITunesMetadataPlugin, + "openlibrary": OpenLibraryMetadataPlugin, + "googlebooks": GoogleBooksMetadataPlugin, + "google": GoogleBooksMetadataPlugin, + "isbnsearch": ISBNsearchMetadataPlugin, + "musicbrainz": MusicBrainzMetadataPlugin, + "imdb": ImdbMetadataPlugin, + "ytdlp": YtdlpMetadataPlugin, + "tidal": TidalMetadataPlugin, + } + + +def register_metadata_plugin(name: str, plugin_cls: Type[MetadataPlugin]) -> None: + _METADATA_PLUGINS[name.lower()] = plugin_cls + + +def list_metadata_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: + availability: Dict[str, + bool] = {} + for name, cls in _METADATA_PLUGINS.items(): + try: + _ = cls(config) + # Basic availability check: perform lightweight validation if defined + availability[name] = True + except Exception: + availability[name] = False + return availability + + +def get_metadata_plugin(name: str, + config: Optional[Dict[str, + Any]] = None + ) -> Optional[MetadataPlugin]: + cls = _METADATA_PLUGINS.get(name.lower()) + if not cls: + return None + try: + return cls(config) + except Exception as exc: + log(f"Metadata plugin init failed for '{name}': {exc}", file=sys.stderr) + return None + + +def get_default_subject_scrape_plugin( + config: Optional[Dict[str, Any]] = None, +) -> Optional[MetadataPlugin]: + best_plugin: Optional[MetadataPlugin] = None + best_priority = 0 + for cls in _METADATA_PLUGINS.values(): + try: + plugin = cls(config) + priority = int(plugin.default_subject_scrape_priority()) + except Exception: + continue + if priority > best_priority: + best_priority = priority + best_plugin = plugin + return best_plugin + + +def get_metadata_plugin_for_url( + url: str, + config: Optional[Dict[str, Any]] = None, +) -> Optional[MetadataPlugin]: + text = str(url or "").strip() + if not text: + return None + + best_plugin: Optional[MetadataPlugin] = None + best_priority = 0 + for cls in _METADATA_PLUGINS.values(): + try: + plugin = cls(config) + priority = int(plugin.url_scrape_priority(text)) + except Exception: + continue + if priority > best_priority: + best_priority = priority + best_plugin = plugin + return best_plugin diff --git a/plugins/scp/__init__.py b/plugins/scp/__init__.py index 0d4c8ef..0ba1309 100644 --- a/plugins/scp/__init__.py +++ b/plugins/scp/__init__.py @@ -16,18 +16,6 @@ from scp import SCPClient from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments -def _pick_provider_config(config: Any) -> Dict[str, Any]: - if not isinstance(config, dict): - return {} - provider = config.get("provider") - if not isinstance(provider, dict): - return {} - entry = provider.get("scp") - if isinstance(entry, dict): - return entry - return {} - - def _coerce_bool(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value @@ -166,20 +154,55 @@ class SCP(Provider): def __init__(self, config: Optional[Dict[str, Any]] = None): super().__init__(config) - conf = _pick_provider_config(self.config) - self._host = str(conf.get("host") or "").strip() - self._port = _coerce_int(conf.get("port"), 22) - self._username = str(conf.get("username") or conf.get("user") or "").strip() - self._password = str(conf.get("password") or "").strip() - self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip() - self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) - self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) - self._allow_agent = _coerce_bool(conf.get("allow_agent"), True) - self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True) - self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") + _instance_name, conf = self.resolve_plugin_instance() + defaults = self._settings_from_config(conf) + self._host = str(defaults.get("host") or "").strip() + self._port = int(defaults.get("port") or 22) + self._username = str(defaults.get("username") or "").strip() + self._password = str(defaults.get("password") or "").strip() + self._key_path = str(defaults.get("key_path") or "").strip() + self._timeout = max(1, int(defaults.get("timeout") or 20)) + self._search_depth = max(0, int(defaults.get("search_depth") or 1)) + self._allow_agent = bool(defaults.get("allow_agent")) + self._look_for_keys = bool(defaults.get("look_for_keys")) + self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/") + + def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]: + entry = dict(conf or {}) + return { + "instance": str(instance_name or entry.get("_instance_name") or "").strip() or None, + "host": str(entry.get("host") or "").strip(), + "port": _coerce_int(entry.get("port"), 22), + "username": str(entry.get("username") or entry.get("user") or "").strip(), + "password": str(entry.get("password") or "").strip(), + "key_path": str(entry.get("key_path") or entry.get("identity_file") or "").strip(), + "timeout": max(1, _coerce_int(entry.get("timeout"), 20)), + "search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)), + "allow_agent": _coerce_bool(entry.get("allow_agent"), True), + "look_for_keys": _coerce_bool(entry.get("look_for_keys"), True), + "base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"), + } + + def _resolve_settings( + self, + *, + filters: Optional[Dict[str, Any]] = None, + instance_name: Optional[str] = None, + require_explicit: bool = False, + ) -> Dict[str, Any]: + requested = self.requested_instance_name(filters, instance=instance_name) + resolved_name, conf = self.resolve_plugin_instance( + requested, + require_explicit=require_explicit or bool(requested), + ) + settings = self._settings_from_config(conf, instance_name=resolved_name) + if settings.get("instance") is None and requested: + settings["instance"] = requested + return settings def validate(self) -> bool: - return bool(self._host and self._username) + settings = self._resolve_settings() + return bool(settings.get("host") and settings.get("username")) def config_helper_text(self) -> str: return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here." @@ -210,6 +233,10 @@ class SCP(Provider): text, inline = parse_inline_query_arguments(query) filters: Dict[str, Any] = {} + instance_name = str(inline.get("instance") or inline.get("store") or "").strip() + if instance_name: + filters["instance"] = instance_name + if inline.get("path"): filters["path"] = inline.get("path") if inline.get("depth"): @@ -220,17 +247,21 @@ class SCP(Provider): return text, filters def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: - active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) + settings = self._resolve_settings(filters=filters) + active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")) + instance_name = str(settings.get("instance") or "").strip() text = str(query or "").strip() if not text or text == "*": - return f"SCP: {active_path}" - return f"SCP: {text} @ {active_path}" + return f"SCP{f'[{instance_name}]' if instance_name else ''}: {active_path}" + return f"SCP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}" def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + settings = self._resolve_settings(filters=filters) return { "plugin": self.name, - "host": self._host, - "path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), + "instance": settings.get("instance"), + "host": settings.get("host"), + "path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")), "query": str(query or "").strip(), } @@ -243,15 +274,21 @@ class SCP(Provider): ) -> List[SearchResult]: _ = kwargs active_filters = dict(filters or {}) - start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) - search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) + settings = self._resolve_settings(filters=active_filters, require_explicit=True) + if not settings.get("host") or not settings.get("username"): + requested = self.requested_instance_name(active_filters) + if requested: + raise RuntimeError(f"SCP instance '{requested}' is unavailable") + return [] + start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")) + search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth))) type_filter = str(active_filters.get("type") or "any").strip().lower() needle = str(query or "").strip() max_results = max(0, int(limit or 0)) if max_results <= 0: return [] - ssh = self._connect_ssh() + ssh = self._connect_ssh(settings) sftp = None try: try: @@ -266,6 +303,7 @@ class SCP(Provider): limit=max_results, search_depth=search_depth, type_filter=type_filter, + settings=settings, ) return self._search_directory( @@ -275,6 +313,7 @@ class SCP(Provider): limit=max_results, search_depth=search_depth, type_filter=type_filter, + settings=settings, ) finally: self._close_client(sftp) @@ -293,19 +332,23 @@ class SCP(Provider): target_path = "" target_title = "" + instance_name = "" for item in selected_items or []: metadata = self._item_metadata(item) if not metadata.get("is_dir"): continue - target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path) + settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance"))) + target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/")) target_title = str(metadata.get("title") or metadata.get("name") or "").strip() + instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip() if target_path: break if not target_path: return False - ssh = self._connect_ssh() + settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name)) + ssh = self._connect_ssh(settings) sftp = None try: try: @@ -320,6 +363,7 @@ class SCP(Provider): limit=500, search_depth=0, type_filter="any", + settings=settings, ) else: rows = self._search_directory( @@ -329,6 +373,7 @@ class SCP(Provider): limit=500, search_depth=0, type_filter="any", + settings=settings, ) finally: self._close_client(sftp) @@ -341,18 +386,23 @@ class SCP(Provider): return True title = target_title or target_path - table = Table(f"SCP: {title}")._perseverance(True) + table = Table(f"SCP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True) table.set_table("scp") try: table.set_table_metadata({ "provider": "scp", - "host": self._host, + "instance": instance_name or None, + "host": settings.get("host"), "path": target_path, "view": "directory", }) except Exception: pass - table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"]) + source_args = ["-plugin", "scp"] + if instance_name: + source_args.extend(["-instance", instance_name]) + source_args.extend([f"path:{target_path}", "*"]) + table.set_source_command("search-file", source_args) payloads: List[Dict[str, Any]] = [] for row in rows: @@ -360,7 +410,7 @@ class SCP(Provider): payloads.append(row.to_dict()) try: - ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path}) + ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "instance": instance_name or None, "path": target_path}) ctx.set_current_stage_table(table) except Exception: pass @@ -373,6 +423,77 @@ class SCP(Provider): return True + def show_selection_details( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + source_command: str = "", + table_type: str = "", + table_metadata: Optional[Dict[str, Any]] = None, + **_kwargs: Any, + ) -> bool: + _ = table_type + item, _payload, _meta = self.resolve_selection_detail_subject( + selected_items, + stage_is_last=stage_is_last, + source_command=source_command, + require_media_kind="file", + ) + if item is None: + return False + + metadata = self._item_metadata(item) + if bool(metadata.get("is_dir")): + return False + + title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "SCP Item" + instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip() + scp_url = str(metadata.get("scp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip() + remote_path = str(metadata.get("scp_path") or "").strip() + host = str(metadata.get("host") or "").strip() + modified = str(metadata.get("modified") or "").strip() + + try: + from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view + except Exception: + return super().show_selection_details( + selected_items, + ctx=ctx, + stage_is_last=stage_is_last, + source_command=source_command, + table_type=table_type, + table_metadata=table_metadata, + ) + + detail_metadata = prepare_detail_metadata( + item, + title=title, + store=instance_name or self.name, + path=scp_url or remote_path or None, + tags=metadata.get("tag") or metadata.get("tags"), + extra_fields={ + "Plugin": self.name, + "Host": host or None, + "Instance": instance_name or None, + "Remote Path": remote_path or None, + "Directory": str(metadata.get("detail") or "").strip() or None, + "Modified": modified or None, + "Scp Url": scp_url or None, + }, + ) + + return render_selection_detail_view( + ctx=ctx, + item=item, + title=f"SCP Item: {title}", + metadata=detail_metadata, + table_name=self.name, + detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "SCP URL", "Plugin"], + value_case="preserve", + ) + def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: metadata = getattr(result, "full_metadata", None) if isinstance(metadata, dict) and metadata.get("is_dir"): @@ -380,10 +501,15 @@ class SCP(Provider): target = str(getattr(result, "path", "") or "").strip() if not target: return None - return self.download_url(target, output_dir, title=getattr(result, "title", None)) + instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else "" + return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None) def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: - settings = self._connection_settings_for_url(url) + parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {} + settings = self._connection_settings_for_url( + url, + instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None, + ) remote_path = settings["path"] if not remote_path or remote_path == "/": return None @@ -431,7 +557,12 @@ class SCP(Provider): return None, None, None temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-")) - downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) + downloaded = self.download_url( + download_url, + temp_dir, + title=metadata.get("title"), + instance=metadata.get("instance"), + ) if downloaded is None: try: temp_dir.rmdir() @@ -451,11 +582,24 @@ class SCP(Provider): if not local_path.exists() or not local_path.is_file(): raise FileNotFoundError(f"File not found: {local_path}") - remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) + settings = self._resolve_settings( + instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None, + require_explicit=bool(kwargs.get("instance") or kwargs.get("store")), + ) + if not settings.get("host") or not settings.get("username"): + requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip() + if requested: + raise RuntimeError(f"SCP instance '{requested}' is unavailable") + raise RuntimeError("No configured SCP instance is available") + + remote_dir = self._normalize_remote_path( + kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/", + default=str(settings.get("base_path") or "/"), + ) remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name remote_path = self._join_remote_path(remote_dir, remote_name) - ssh = self._connect_ssh() + ssh = self._connect_ssh(settings) sftp = None scp_client = None try: @@ -466,7 +610,7 @@ class SCP(Provider): raise self._ensure_directory_via_ssh(ssh, remote_dir) else: - self._ensure_directory(sftp, remote_dir) + self._ensure_directory(sftp, remote_dir, base_path=str(settings.get("base_path") or "/")) scp_client = self._open_scp(ssh) scp_client.put(str(local_path), remote_path=remote_path) finally: @@ -474,19 +618,20 @@ class SCP(Provider): self._close_client(sftp) self._close_client(ssh) - return self._build_url(remote_path) + return self._build_url(remote_path, settings=settings) def _run_test_connection(self) -> Dict[str, Any]: - if not self._host: + settings = self._resolve_settings() + if not settings.get("host"): return {"ok": False, "message": "Set 'host' before testing the SCP connection."} - if not self._username: + if not settings.get("username"): return {"ok": False, "message": "Set 'username' before testing the SCP connection."} ssh = None sftp = None try: - ssh = self._connect_ssh() - base_path = self._base_path or "/" + ssh = self._connect_ssh(settings) + base_path = str(settings.get("base_path") or "/") transport_detail = "SFTP available" try: sftp = self._open_sftp(ssh) @@ -502,10 +647,11 @@ class SCP(Provider): except Exception: is_dir = False detail = f" and confirmed {base_path}" if is_dir else "" - auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth" + key_path = str(settings.get("key_path") or "").strip() + auth_mode = f"key {key_path}" if key_path else "password/agent auth" return { "ok": True, - "message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.", + "message": f"Connected to SCP {settings.get('host')}:{settings.get('port')} as {settings.get('username')} via {auth_mode}. {transport_detail}{detail}.", } except Exception as exc: return {"ok": False, "message": f"SCP connection failed: {exc}"} @@ -514,7 +660,9 @@ class SCP(Provider): self._close_client(ssh) def _generate_ssh_keypair(self) -> Dict[str, Any]: - target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa") + settings = self._resolve_settings() + key_path = str(settings.get("key_path") or "").strip() + target = Path(key_path).expanduser() if key_path else (Path.home() / ".ssh" / "medeia_scp_rsa") try: target.parent.mkdir(parents=True, exist_ok=True) except Exception as exc: @@ -530,7 +678,7 @@ class SCP(Provider): try: key = paramiko.RSAKey.generate(bits=4096) key.write_private_key_file(str(target)) - comment = f"{self._username or 'medeia'}@{self._host or 'scp'}" + comment = f"{settings.get('username') or 'medeia'}@{settings.get('host') or 'scp'}" public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8") try: target.chmod(0o600) @@ -654,34 +802,40 @@ class SCP(Provider): self, remote_path: Any, *, + settings: Optional[Dict[str, Any]] = None, host: Optional[str] = None, port: Optional[int] = None, scheme: str = "scp", ) -> str: + resolved = dict(settings or {}) path_text = self._normalize_remote_path(remote_path, default="/") - host_text = str(host or self._host).strip() - port_value = int(port or self._port) + host_text = str(host or resolved.get("host") or self._host).strip() + port_value = int(port or resolved.get("port") or self._port) port_suffix = f":{port_value}" if port_value and port_value != 22 else "" return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" - def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: + def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]: + settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name)) parsed = urlparse(str(url or "").strip()) scheme = (parsed.scheme or "scp").strip().lower() - host = parsed.hostname or self._host - port = parsed.port or self._port - username = parsed.username or self._username - password = parsed.password or self._password - path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") + host = parsed.hostname or settings.get("host") or self._host + port = parsed.port or settings.get("port") or self._port + username = parsed.username or settings.get("username") or self._username + password = parsed.password or settings.get("password") or self._password + path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/")) return { + "instance": settings.get("instance"), "scheme": scheme, "host": host, "port": port, "username": username, "password": password, - "key_path": self._key_path, - "allow_agent": self._allow_agent, - "look_for_keys": self._look_for_keys, + "key_path": settings.get("key_path") or self._key_path, + "allow_agent": settings.get("allow_agent", self._allow_agent), + "look_for_keys": settings.get("look_for_keys", self._look_for_keys), "path": path_text, + "timeout": settings.get("timeout", self._timeout), + "base_path": settings.get("base_path", self._base_path), } def _search_directory( @@ -693,12 +847,13 @@ class SCP(Provider): limit: int, search_depth: int, type_filter: str, + settings: Dict[str, Any], ) -> List[SearchResult]: results: List[SearchResult] = [] visited: set[str] = set() def walk(current_path: str, depth_left: int) -> None: - normalized = self._normalize_remote_path(current_path, default=self._base_path) + normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path)) if normalized in visited or len(results) >= limit: return visited.add(normalized) @@ -707,7 +862,7 @@ class SCP(Provider): if len(results) >= limit: return if self._matches_entry(entry, needle=needle, type_filter=type_filter): - results.append(self._build_result(entry)) + results.append(self._build_result(entry, settings=settings)) if entry.get("is_dir") and depth_left > 0: walk(str(entry.get("scp_path") or normalized), depth_left - 1) @@ -723,6 +878,7 @@ class SCP(Provider): limit: int, search_depth: int, type_filter: str, + settings: Dict[str, Any], ) -> List[SearchResult]: entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth) results: List[SearchResult] = [] @@ -730,7 +886,7 @@ class SCP(Provider): if len(results) >= limit: break if self._matches_entry(entry, needle=needle, type_filter=type_filter): - results.append(self._build_result(entry)) + results.append(self._build_result(entry, settings=settings)) return results def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool: @@ -756,16 +912,18 @@ class SCP(Provider): return False return True - def _build_result(self, entry: Dict[str, Any]) -> SearchResult: + def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult: scp_path = str(entry.get("scp_path") or "/") - scp_url = self._build_url(scp_path) + scp_url = self._build_url(scp_path, settings=settings) is_dir = bool(entry.get("is_dir")) size_value = entry.get("size") modified = str(entry.get("modified") or "") parent = posixpath.dirname(scp_path.rstrip("/")) or "/" + instance_name = str(settings.get("instance") or "").strip() metadata = { "provider": "scp", - "host": self._host, + "instance": instance_name or None, + "host": settings.get("host"), "scp_path": scp_path, "scp_url": scp_url, "selection_url": scp_url, @@ -777,6 +935,13 @@ class SCP(Provider): if modified: metadata["modified"] = modified + selection_args = ["-url", scp_url] + selection_action = ["download-file", "-plugin", "scp"] + if instance_name: + selection_args = ["-instance", instance_name, *selection_args] + selection_action.extend(["-instance", instance_name]) + selection_action.extend(["-url", scp_url]) + return SearchResult( table="scp", title=str(entry.get("name") or scp_path), @@ -793,8 +958,8 @@ class SCP(Provider): ("Size", "" if size_value is None else str(size_value)), ("Modified", modified), ], - selection_args=None if is_dir else ["-url", scp_url], - selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url], + selection_args=None if is_dir else selection_args, + selection_action=None if is_dir else selection_action, full_metadata=metadata, ) @@ -867,8 +1032,8 @@ class SCP(Provider): ) return entries - def _ensure_directory(self, sftp: Any, remote_path: str) -> None: - normalized = self._normalize_remote_path(remote_path, default=self._base_path) + def _ensure_directory(self, sftp: Any, remote_path: str, *, base_path: str) -> None: + normalized = self._normalize_remote_path(remote_path, default=base_path) if normalized == "/": return partial = "" @@ -923,11 +1088,18 @@ class SCP(Provider): if path_text.startswith(("scp://", "sftp://")): scp_path = self._normalize_remote_path(path_text, default=self._base_path) if scp_path: - metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path) + base_path = str(metadata.get("base_path") or self._base_path) + metadata["scp_path"] = self._normalize_remote_path(scp_path, default=base_path) metadata.setdefault("selection_path", metadata["scp_path"]) if metadata.get("scp_path") and not metadata.get("scp_url"): - metadata["scp_url"] = self._build_url(metadata["scp_path"]) + metadata["scp_url"] = self._build_url( + metadata["scp_path"], + settings={ + "host": metadata.get("host") or self._host, + "instance": metadata.get("instance"), + }, + ) if metadata.get("scp_url") and not metadata.get("selection_url"): metadata["selection_url"] = metadata["scp_url"] diff --git a/plugins/tidal_manifest.py b/plugins/tidal_manifest.py index 0b4d242..00dcb1e 100644 --- a/plugins/tidal_manifest.py +++ b/plugins/tidal_manifest.py @@ -1,7 +1,343 @@ -"""Plugin-namespace import shim for Tidal/HIFI manifest helpers. +"""Tidal/HIFI manifest helpers. -The implementation currently lives in ``Provider.tidal_manifest`` while the -legacy namespace is phased out. New imports should prefer ``plugins``. +This module intentionally lives with the provider code (not cmdlets). +It contains best-effort helpers for turning proxy-provided Tidal "manifest" +values into a playable input reference: +- A local MPD file path (persisted to temp) +- Or a direct URL (when the manifest is JSON with `urls`) + +Callers may pass either a SearchResult-like object (with `.full_metadata`) or +pipeline dicts. """ -from Provider.tidal_manifest import * # noqa: F401,F403 +from __future__ import annotations + +import base64 +import hashlib +import json +import re +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, Optional + +from API.httpx_shared import get_shared_httpx_client +from SYS.logger import log + + +_DEFAULT_TIDAL_TRACK_API_BASES = ( + "https://triton.squid.wtf", + "https://wolf.qqdl.site", + "https://maus.qqdl.site", + "https://vogel.qqdl.site", + "https://katze.qqdl.site", + "https://hund.qqdl.site", + "https://tidal.kinoplus.online", + "https://tidal-api.binimum.org", +) + + +def resolve_tidal_manifest_path(item: Any) -> Optional[str]: + """Persist the Tidal manifest (MPD) and return a local path or URL. + + Resolution order: + 1) `_tidal_manifest_path` (existing local file) + 2) `_tidal_manifest_url` (existing remote URL) + 3) decode `manifest` and: + - if JSON with `urls`: return the first URL + - if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path + + If `manifest` is missing but a track id exists, the function will attempt a + best-effort fetch from the public proxy endpoints to populate `manifest`. + """ + + metadata: Any = None + if isinstance(item, dict): + metadata = item.get("full_metadata") or item.get("metadata") + else: + metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None) + + if not isinstance(metadata, dict): + return None + + existing_path = metadata.get("_tidal_manifest_path") + if existing_path: + try: + resolved = Path(str(existing_path)) + if resolved.is_file(): + return str(resolved) + except Exception: + pass + + existing_url = metadata.get("_tidal_manifest_url") + if existing_url and isinstance(existing_url, str): + candidate = existing_url.strip() + if candidate: + return candidate + + raw_manifest = metadata.get("manifest") + if not raw_manifest: + _maybe_fetch_track_manifest(item, metadata) + raw_manifest = metadata.get("manifest") + if not raw_manifest: + return None + + manifest_str = "".join(str(raw_manifest or "").split()) + if not manifest_str: + return None + + manifest_bytes: bytes + try: + manifest_bytes = base64.b64decode(manifest_str, validate=True) + except Exception: + try: + manifest_bytes = base64.b64decode(manifest_str, validate=False) + except Exception: + try: + manifest_bytes = manifest_str.encode("utf-8") + except Exception: + return None + + if not manifest_bytes: + return None + + head = (manifest_bytes[:1024] or b"").lstrip() + if head.startswith((b"{", b"[")): + return _resolve_json_manifest_urls(metadata, manifest_bytes) + + looks_like_mpd = head.startswith((b" Optional[str]: + text = str(candidate or "").strip() + if not text: + return None + if not re.match(r"^https?://", text, flags=re.IGNORECASE): + return None + return text.rstrip("/") + + +def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]: + bases: list[str] = [] + seen: set[str] = set() + + dynamic_candidates = [ + metadata.get("_tidal_api_base"), + metadata.get("_api_base"), + metadata.get("api_base"), + metadata.get("base_url"), + ] + + for candidate in dynamic_candidates: + normalized = _normalize_api_base(candidate) + if normalized and normalized not in seen: + seen.add(normalized) + bases.append(normalized) + + for candidate in _DEFAULT_TIDAL_TRACK_API_BASES: + normalized = _normalize_api_base(candidate) + if normalized and normalized not in seen: + seen.add(normalized) + bases.append(normalized) + + return bases + + +def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None: + """If we only have a track id, fetch details from the proxy to populate `manifest`.""" + + try: + already = bool(metadata.get("_tidal_track_details_fetched")) + except Exception: + already = False + + track_id = metadata.get("trackId") or metadata.get("id") + + if track_id is None: + try: + if isinstance(item, dict): + candidate_path = item.get("path") or item.get("url") + else: + candidate_path = getattr(item, "path", None) or getattr(item, "url", None) + except Exception: + candidate_path = None + + if candidate_path: + m = re.search( + r"(tidal|hifi):(?://)?track[\\/](\d+)", + str(candidate_path), + flags=re.IGNORECASE, + ) + if m: + track_id = m.group(2) + + if already or track_id is None: + return + + try: + track_int = int(track_id) + except Exception: + track_int = None + + if not track_int or track_int <= 0: + return + + try: + client = get_shared_httpx_client() + except Exception: + return + + attempted = False + for base in _iter_track_api_bases(metadata): + attempted = True + + track_data: Optional[Dict[str, Any]] = None + for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}): + try: + resp = client.get( + f"{base}/track/", + params=params, + timeout=10.0, + ) + resp.raise_for_status() + payload = resp.json() + data = payload.get("data") if isinstance(payload, dict) else None + if isinstance(data, dict) and data: + track_data = data + break + except Exception: + continue + + if isinstance(track_data, dict) and track_data: + try: + metadata.update(track_data) + except Exception: + pass + + if not metadata.get("manifest") or not metadata.get("url"): + try: + resp_info = client.get( + f"{base}/info/", + params={"id": str(track_int)}, + timeout=10.0, + ) + resp_info.raise_for_status() + info_payload = resp_info.json() + info_data = info_payload.get("data") if isinstance(info_payload, dict) else None + if isinstance(info_data, dict) and info_data: + try: + for key, value in info_data.items(): + if key not in metadata or not metadata.get(key): + metadata[key] = value + except Exception: + pass + except Exception: + pass + + if metadata.get("manifest"): + break + + if attempted: + try: + metadata["_tidal_track_details_fetched"] = True + except Exception: + pass + + +def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]: + try: + text = manifest_bytes.decode("utf-8", errors="ignore") + payload = json.loads(text) + urls = payload.get("urls") or [] + selected_url = None + for candidate in urls: + if isinstance(candidate, str): + candidate = candidate.strip() + if candidate: + selected_url = candidate + break + if selected_url: + try: + metadata["_tidal_manifest_url"] = selected_url + except Exception: + pass + return selected_url + try: + metadata["_tidal_manifest_error"] = "JSON manifest contained no urls" + except Exception: + pass + log( + f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls", + file=sys.stderr, + ) + except Exception as exc: + try: + metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}" + except Exception: + pass + log( + f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}", + file=sys.stderr, + ) + return None + + +def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]: + manifest_hash = str(metadata.get("manifestHash") or "").strip() + track_id = metadata.get("trackId") or metadata.get("id") + + identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest() + identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64] + if not identifier_safe: + identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12] + + track_safe = "tidal" + if track_id is not None: + track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal" + + manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal" + try: + manifest_dir.mkdir(parents=True, exist_ok=True) + except Exception: + pass + + filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd" + target_path = manifest_dir / filename + + try: + with open(target_path, "wb") as fh: + fh.write(manifest_bytes) + metadata["_tidal_manifest_path"] = str(target_path) + + # Best-effort: propagate back into the caller object/dict. + if isinstance(item, dict): + if item.get("full_metadata") is metadata: + item["full_metadata"] = metadata + elif item.get("metadata") is metadata: + item["metadata"] = metadata + else: + extra = getattr(item, "extra", None) + if isinstance(extra, dict): + extra["_tidal_manifest_path"] = str(target_path) + + return str(target_path) + except Exception: + return None