updating and refining plugin system refactor

This commit is contained in:
2026-04-28 22:20:54 -07:00
parent 8685fbb723
commit 323c24f4f4
33 changed files with 4287 additions and 3312 deletions
+2 -33
View File
@@ -1,37 +1,6 @@
"""Hydrus API helpers and export utilities.""" """Compatibility shim for the Hydrus plugin-owned API module."""
from __future__ import annotations from plugins.hydrusnetwork.api import * # noqa: F401,F403
import base64
import http.client
import json
import os
import re
import shutil
import subprocess
import sys
import time
from collections import deque
from SYS.logger import log
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS as GLOBAL_SUPPORTED_EXTENSIONS
import tempfile
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast
from urllib.parse import urlsplit, urlencode, quote, urlunsplit, unquote
import httpx
logger = logging.getLogger(__name__)
from SYS.utils import (
decode_cbor,
jsonify,
ensure_directory,
unique_path,
)
from .HTTP import HTTPClient
class HydrusRequestError(RuntimeError): class HydrusRequestError(RuntimeError):
+109 -20
View File
@@ -506,32 +506,28 @@ class CmdletIntrospection:
if normalized_arg == "plugin": if normalized_arg == "plugin":
canonical_cmd = (cmd_name or "").replace("_", "-").lower() canonical_cmd = (cmd_name or "").replace("_", "-").lower()
try: try:
from ProviderCore.registry import ( from ProviderCore.registry import list_plugin_names_with_capability
list_search_plugin_names,
list_upload_plugin_names,
)
except Exception: except Exception:
list_search_plugin_names = None # type: ignore list_plugin_names_with_capability = None # type: ignore
list_upload_plugin_names = None # type: ignore
provider_choices: List[str] = [] plugin_choices: List[str] = []
if canonical_cmd in {"add-file"} and list_upload_plugin_names is not None: if canonical_cmd in {"add-file"} and list_plugin_names_with_capability is not None:
return list_upload_plugin_names() or [] return list_plugin_names_with_capability("upload") or []
if list_search_plugin_names is not None: if list_plugin_names_with_capability is not None:
provider_choices = list_search_plugin_names() or [] plugin_choices = list_plugin_names_with_capability("search") or []
if provider_choices: if plugin_choices:
return provider_choices return plugin_choices
if normalized_arg == "scrape": if normalized_arg == "scrape":
try: try:
from plugins.metadata_provider import list_metadata_providers from plugins.metadata_provider import list_metadata_plugins
meta_providers = list_metadata_providers(config) or {} metadata_plugins = list_metadata_plugins(config) or {}
if meta_providers: if metadata_plugins:
return sorted(meta_providers.keys()) return sorted(metadata_plugins.keys())
except Exception: except Exception:
pass pass
@@ -704,6 +700,77 @@ class CmdletCompleter(Completer):
return tokens[idx + 1] return tokens[idx + 1]
return None return None
@staticmethod
def _selected_plugin_name(cmd_name: str, stage_tokens: Sequence[str]) -> Optional[str]:
canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower()
if canonical_cmd not in {"search-file", "add-file", "download-file"}:
return None
return CmdletCompleter._flag_value(stage_tokens, "-plugin", "--plugin")
@staticmethod
def _plugin_instance_choices(plugin_name: Optional[str], config: Dict[str, Any]) -> List[str]:
plugin_key = str(plugin_name or "").strip().lower()
if not plugin_key:
return []
try:
from ProviderCore.registry import get_plugin_class
except Exception:
return []
plugin_class = get_plugin_class(plugin_key)
if plugin_class is None:
return []
try:
plugin = plugin_class(config)
except Exception:
return []
try:
instances = plugin.configured_instances()
except Exception:
return []
out: List[str] = []
seen: Set[str] = set()
for value in instances or []:
text = str(value or "").strip()
lowered = text.lower()
if not text or lowered in seen:
continue
seen.add(lowered)
out.append(text)
return out
def _filter_stage_arg_names(
self,
*,
cmd_name: str,
stage_tokens: Sequence[str],
config: Dict[str, Any],
arg_names: List[str],
) -> List[str]:
if not arg_names:
return []
canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower()
plugin_name = self._selected_plugin_name(canonical_cmd, stage_tokens)
instance_choices = self._plugin_instance_choices(plugin_name, config)
has_named_instances = bool(instance_choices)
filtered: List[str] = []
for arg in arg_names:
logical = str(arg or "").lstrip("-").strip().lower()
if logical == "instance":
if not plugin_name or not has_named_instances:
continue
if canonical_cmd == "search-file" and logical == "open":
if str(plugin_name or "").strip().lower() != "alldebrid":
continue
filtered.append(arg)
return filtered
def get_completions( def get_completions(
self, self,
document: Document, document: Document,
@@ -742,7 +809,12 @@ class CmdletCompleter(Completer):
if cmd_name not in self.cmdlet_names: if cmd_name not in self.cmdlet_names:
return return
arg_names = self._cmdlet_args(cmd_name, config) arg_names = self._filter_stage_arg_names(
cmd_name=cmd_name,
stage_tokens=stage_tokens,
config=config,
arg_names=self._cmdlet_args(cmd_name, config),
)
seen_logicals: Set[str] = set() seen_logicals: Set[str] = set()
for arg in arg_names: for arg in arg_names:
arg_low = arg.lower() arg_low = arg.lower()
@@ -779,6 +851,8 @@ class CmdletCompleter(Completer):
if cmd_name == "search-file": if cmd_name == "search-file":
provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin") provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin")
selected_plugin = self._selected_plugin_name(cmd_name, stage_tokens)
query_specs = self._query_args(cmd_name, config) query_specs = self._query_args(cmd_name, config)
query_flag_index = -1 query_flag_index = -1
for idx, tok in enumerate(stage_tokens): for idx, tok in enumerate(stage_tokens):
@@ -886,6 +960,11 @@ class CmdletCompleter(Completer):
yield Completion(suggestion, start_position=start_pos) yield Completion(suggestion, start_position=start_pos)
return return
normalized_prev = prev_token.lstrip("-").strip().lower()
choices: List[str] = []
if normalized_prev == "instance" and selected_plugin:
choices = self._plugin_instance_choices(selected_plugin, config)
if not choices:
choices = self._arg_choices( choices = self._arg_choices(
cmd_name=cmd_name, cmd_name=cmd_name,
arg_name=prev_token, arg_name=prev_token,
@@ -894,20 +973,30 @@ class CmdletCompleter(Completer):
) )
if choices: if choices:
choice_list = choices choice_list = choices
normalized_prev = prev_token.lstrip("-").strip().lower()
if normalized_prev in {"plugin", "provider"} and current_token: if normalized_prev in {"plugin", "provider"} and current_token:
current_lower = current_token.lower() current_lower = current_token.lower()
filtered = [c for c in choices if current_lower in c.lower()] filtered = [c for c in choices if current_lower in c.lower()]
if filtered: if filtered:
choice_list = filtered choice_list = filtered
if normalized_prev == "instance" and current_token:
current_lower = current_token.lower()
filtered = [c for c in choice_list if current_lower in c.lower()]
if filtered:
choice_list = filtered
for choice in choice_list: for choice in choice_list:
yield Completion(choice, start_position=-len(current_token)) yield Completion(choice, start_position=-len(current_token))
# Example: if the user has typed `download-file -url ...`, then `url` # Example: if the user has typed `download-file -url ...`, then `url`
# is considered used and should not be suggested again (even as `--url`). # is considered used and should not be suggested again (even as `--url`).
return return
arg_names = self._cmdlet_args(cmd_name, config) arg_names = self._filter_stage_arg_names(
cmd_name=cmd_name,
stage_tokens=stage_tokens,
config=config,
arg_names=self._cmdlet_args(cmd_name, config),
)
used_logicals = self._used_arg_logicals(cmd_name, stage_tokens, config) used_logicals = self._used_arg_logicals(cmd_name, stage_tokens, config)
logical_seen: Set[str] = set() logical_seen: Set[str] = set()
for arg in arg_names: for arg in arg_names:
+6
View File
@@ -4105,6 +4105,9 @@ function M._apply_web_subtitle_load_defaults(reason)
if target == '' or not _is_http_url(target) then if target == '' or not _is_http_url(target) then
return false return false
end end
if not _is_ytdlp_url(target) then
return false
end
M._prepare_ytdl_format_for_web_load(target, reason or 'on-load') M._prepare_ytdl_format_for_web_load(target, reason or 'on-load')
@@ -4250,6 +4253,9 @@ function M._ensure_current_subtitles_visible(reason)
if (not current or current == '') and (path == '' or not _is_http_url(path)) then if (not current or current == '') and (path == '' or not _is_http_url(path)) then
return false return false
end end
if (not current or current == '') and (path == '' or not _is_ytdlp_url(path)) then
return false
end
local track_id, source, already_selected = M._find_subtitle_track_candidate() local track_id, source, already_selected = M._find_subtitle_track_candidate()
if not track_id then if not track_id then
+5 -256
View File
@@ -1,259 +1,8 @@
"""Example plugin that uses the new `ResultTable` API. """Legacy compatibility shim for the strict adapter example module.
This module demonstrates a minimal provider adapter that yields `ResultModel` The active implementation now lives in ``plugins.example_provider`` so the
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer plugin namespace owns the example adapter module. Keep this file only to avoid
(`render_table`) for demonstration. breaking old imports while the legacy ``Provider`` package is phased out.
Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default
""" """
from __future__ import annotations
from pathlib import Path from plugins.example_provider import * # noqa: F401,F403
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column
SAMPLE_ITEMS = [
{
"name": "Book of Awe.pdf",
"path": "sample/Book of Awe.pdf",
"ext": "pdf",
"size": 1024000,
"source": "example",
},
{
"name": "Song of Joy.mp3",
"path": "sample/Song of Joy.mp3",
"ext": "mp3",
"size": 5120000,
"source": "example",
},
{
"name": "Cover Image.jpg",
"path": "sample/Cover Image.jpg",
"ext": "jpg",
"size": 20480,
"source": "example",
},
]
def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]:
"""Convert provider-specific items into `ResultModel` instances.
This adapter enforces the strict API requirement: it yields only
`ResultModel` instances (no legacy dict objects).
"""
for it in items:
title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "")
yield ResultModel(
title=str(title),
path=str(it.get("path")) if it.get("path") else None,
ext=str(it.get("ext")) if it.get("ext") else None,
size_bytes=int(it.get("size")) if it.get("size") is not None else None,
metadata=dict(it),
source=str(it.get("source")) if it.get("source") else "example",
)
# Columns are intentionally *not* mandated. Create a factory that inspects
# sample rows and builds only columns that make sense for the provider data.
from SYS.result_table_api import metadata_column
def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]:
cols: List[ColumnSpec] = [title_column()]
# If any row has an extension, include Ext column
if any(getattr(r, "ext", None) for r in rows):
cols.append(ext_column())
# If any row has size, include Size column
if any(getattr(r, "size_bytes", None) for r in rows):
cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v)))
# Add any top-level metadata keys discovered (up to 3) as optional columns
seen_keys = []
for r in rows:
for k in (r.metadata or {}).keys():
if k in ("name", "title", "path"):
continue
if k not in seen_keys:
seen_keys.append(k)
if len(seen_keys) >= 3:
break
if len(seen_keys) >= 3:
break
for k in seen_keys:
cols.append(metadata_column(k))
return cols
# Selection function: cmdlets rely on this to build selector args when the user
# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer
# -path if available, otherwise fall back to -title.
def selection_fn(row: ResultModel) -> List[str]:
if row.path:
return ["-path", row.path]
return ["-title", row.title]
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
selection_fn=selection_fn,
metadata={"description": "Example provider demonstrating dynamic columns and selectors"},
)
def _format_size(size: Any) -> str:
try:
s = int(size)
except Exception:
return ""
if s >= 1024 ** 3:
return f"{s / (1024 ** 3):.2f} GB"
if s >= 1024 ** 2:
return f"{s / (1024 ** 2):.2f} MB"
if s >= 1024:
return f"{s / 1024:.2f} KB"
return f"{s} B"
def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str:
"""Render a simple ASCII table of `rows` using `columns`.
This is intentionally very small and dependency-free for demonstration.
Renderers in the project should implement the `Renderer` protocol.
"""
rows = list(rows)
# Build cell matrix (strings)
matrix: List[List[str]] = []
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
matrix.append(cells)
# Compute column widths as max(header, content)
headers = [c.header for c in columns]
widths = [len(h) for h in headers]
for row_cells in matrix:
for i, cell in enumerate(row_cells):
widths[i] = max(widths[i], len(cell))
# Helper to format a row
def fmt_row(cells: List[str]) -> str:
return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
lines: List[str] = []
lines.append(fmt_row(headers))
lines.append("-+-".join("-" * w for w in widths))
for row_cells in matrix:
lines.append(fmt_row(row_cells))
return "\n".join(lines)
# Rich-based renderer (returns a Rich Table renderable)
def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]):
"""Render rows as a `rich.table.Table` for terminal output.
Returns the Table object; callers may `Console.print(table)` to render.
"""
try:
from rich.table import Table as RichTable
except Exception as exc: # pragma: no cover - rare if rich missing
raise RuntimeError("rich is required for rich renderer") from exc
table = RichTable(show_header=True, header_style="bold")
for col in columns:
table.add_column(col.header)
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
table.add_row(*cells)
return table
def demo() -> None:
rows = list(adapter(SAMPLE_ITEMS))
table = render_table_rich(rows, columns_factory(rows))
try:
from rich.console import Console
except Exception:
# Fall back to plain printing if rich is not available
print("Example provider output:")
print(render_table(rows, columns_factory(rows)))
return
console = Console()
console.print("Example provider output:")
console.print(table)
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_plugin
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
# Render
try:
from rich.console import Console
except Exception:
print(render_table(rows, cols))
sel_args = provider.selection_args(rows[idx])
print("Selection args for row", idx, "->", sel_args)
return
console = Console()
console.print("Example provider output:")
console.print(render_table_rich(rows, cols))
# Selection args example
sel = provider.selection_args(rows[idx])
console.print("Selection args for row", idx, "->", sel)
if __name__ == "__main__":
demo()
File diff suppressed because it is too large Load Diff
+5 -340
View File
@@ -1,343 +1,8 @@
"""Tidal/HIFI manifest helpers. """Legacy compatibility shim for Tidal manifest helpers.
This module intentionally lives with the provider code (not cmdlets). The active implementation now lives in ``plugins.tidal_manifest`` so the
It contains best-effort helpers for turning proxy-provided Tidal "manifest" plugin namespace owns the manifest helper module. Keep this file only to avoid
values into a playable input reference: breaking old imports while the legacy ``Provider`` package is phased out.
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
""" """
from __future__ import annotations from plugins.tidal_manifest import * # noqa: F401,F403
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
_DEFAULT_TIDAL_TRACK_API_BASES = (
"https://triton.squid.wtf",
"https://wolf.qqdl.site",
"https://maus.qqdl.site",
"https://vogel.qqdl.site",
"https://katze.qqdl.site",
"https://hund.qqdl.site",
"https://tidal.kinoplus.online",
"https://tidal-api.binimum.org",
)
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _normalize_api_base(candidate: Any) -> Optional[str]:
text = str(candidate or "").strip()
if not text:
return None
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
return None
return text.rstrip("/")
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
bases: list[str] = []
seen: set[str] = set()
dynamic_candidates = [
metadata.get("_tidal_api_base"),
metadata.get("_api_base"),
metadata.get("api_base"),
metadata.get("base_url"),
]
for candidate in dynamic_candidates:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
return bases
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
except Exception:
return
attempted = False
for base in _iter_track_api_bases(metadata):
attempted = True
track_data: Optional[Dict[str, Any]] = None
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
try:
resp = client.get(
f"{base}/track/",
params=params,
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
track_data = data
break
except Exception:
continue
if isinstance(track_data, dict) and track_data:
try:
metadata.update(track_data)
except Exception:
pass
if not metadata.get("manifest") or not metadata.get("url"):
try:
resp_info = client.get(
f"{base}/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for key, value in info_data.items():
if key not in metadata or not metadata.get(key):
metadata[key] = value
except Exception:
pass
except Exception:
pass
if metadata.get("manifest"):
break
if attempted:
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None
+232 -33
View File
@@ -114,7 +114,7 @@ def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]:
class Provider(ABC): class Provider(ABC):
"""Unified plugin base class. """Unified plugin base class.
This replaces the older split between search and upload providers. This replaces the older split between search and upload plugins.
Concrete plugins may implement any subset of: Concrete plugins may implement any subset of:
- search(query, ...) - search(query, ...)
- download(result, output_dir) - download(result, output_dir)
@@ -127,8 +127,8 @@ class Provider(ABC):
PLUGIN_NAME: str = "" PLUGIN_NAME: str = ""
PLUGIN_ALIASES: Sequence[str] = () PLUGIN_ALIASES: Sequence[str] = ()
# Optional provider-driven defaults for what to do when a user selects @N from a # Optional plugin-driven defaults for what to do when a user selects @N from a
# provider table. The CLI uses this to auto-insert stages (e.g. download-file) # plugin table. The CLI uses this to auto-insert stages (e.g. download-file)
# without hardcoding table names. # without hardcoding table names.
# #
# Example: # Example:
@@ -138,7 +138,7 @@ class Provider(ABC):
TABLE_AUTO_PREFIXES: Dict[str, Sequence[str]] = {} TABLE_AUTO_PREFIXES: Dict[str, Sequence[str]] = {}
AUTO_STAGE_USE_SELECTION_ARGS: bool = False AUTO_STAGE_USE_SELECTION_ARGS: bool = False
# Optional provider-declared configuration keys. # Optional plugin-declared configuration keys.
# Used for dynamically generating config panels (e.g., missing credentials). # Used for dynamically generating config panels (e.g., missing credentials).
REQUIRED_CONFIG_KEYS: Sequence[str] = () REQUIRED_CONFIG_KEYS: Sequence[str] = ()
@@ -176,7 +176,7 @@ class Provider(ABC):
return False return False
def get_table_type(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: def get_table_type(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
"""Return the table type identifier for results from this provider.""" """Return the table type identifier for results from this plugin."""
return self.name return self.name
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
@@ -197,12 +197,12 @@ class Provider(ABC):
@property @property
def prefers_transfer_progress(self) -> bool: def prefers_transfer_progress(self) -> bool:
"""True if this provider prefers explicit transfer progress tracking (begin/finish) during download.""" """True if this plugin prefers explicit transfer progress tracking (begin/finish) during download."""
return False return False
@classmethod @classmethod
def config_schema(cls) -> List[Dict[str, Any]]: def config_schema(cls) -> List[Dict[str, Any]]:
"""Return configuration schema for this provider. """Return configuration schema for this plugin.
Returns a list of dicts, each defining a field: Returns a list of dicts, each defining a field:
{ {
@@ -234,8 +234,124 @@ class Provider(ABC):
return [] return []
return out return out
@classmethod
def plugin_config_key(cls) -> str:
return str(getattr(cls, "PLUGIN_NAME", None) or cls.__name__ or "").strip().lower()
@classmethod
def plugin_instance_filter_keys(cls) -> Tuple[str, ...]:
return ("instance", "store")
@classmethod
def plugin_config_field_keys(cls) -> set[str]:
keys: set[str] = set()
try:
for field in cls.config_schema() or []:
if not isinstance(field, dict):
continue
key = str(field.get("key") or "").strip().lower()
if key:
keys.add(key)
except Exception:
return set()
return keys
def requested_instance_name(
self,
filters: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Optional[str]:
for key in self.plugin_instance_filter_keys():
value = kwargs.get(key)
if value in (None, "") and isinstance(filters, dict):
value = filters.get(key)
text = str(value or "").strip()
if text:
return text
return None
def plugin_config_root(self) -> Dict[str, Any]:
if not isinstance(self.config, dict):
return {}
provider_cfg = self.config.get("provider")
if not isinstance(provider_cfg, dict):
return {}
entry = provider_cfg.get(self.plugin_config_key())
return dict(entry) if isinstance(entry, dict) else {}
def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]:
entry = self.plugin_config_root()
if not entry:
return {}
schema_keys = self.plugin_config_field_keys()
entry_keys = {str(key or "").strip().lower() for key in entry.keys()}
looks_like_single = bool(schema_keys and entry_keys.intersection(schema_keys))
if not looks_like_single and entry:
looks_like_single = not all(isinstance(value, dict) for value in entry.values())
if looks_like_single:
return {"default": dict(entry)}
instances: Dict[str, Dict[str, Any]] = {}
for raw_name, raw_value in entry.items():
if not isinstance(raw_value, dict):
continue
name = str(raw_name or "").strip()
if not name:
continue
instances[name] = dict(raw_value)
return instances
def configured_instances(self) -> List[str]:
instances = self.plugin_instance_configs()
if not instances:
return []
if set(instances.keys()) == {"default"}:
return []
return list(instances.keys())
def resolve_plugin_instance(
self,
instance_name: Optional[str] = None,
*,
require_explicit: bool = False,
) -> Tuple[Optional[str], Dict[str, Any]]:
instances = self.plugin_instance_configs()
if not instances:
return None, {}
requested = str(instance_name or "").strip()
if not requested:
first_name = next(iter(instances.keys()))
resolved = dict(instances[first_name])
if first_name != "default":
resolved.setdefault("_instance_name", first_name)
return (None if first_name == "default" else first_name), resolved
requested_lower = requested.lower()
for name, cfg in instances.items():
aliases = {str(name).strip().lower()}
explicit_name = str(cfg.get("_instance_name") or cfg.get("instance") or cfg.get("name") or "").strip().lower()
if explicit_name:
aliases.add(explicit_name)
if requested_lower in aliases:
resolved = dict(cfg)
if name != "default":
resolved.setdefault("_instance_name", name)
return (None if name == "default" else name), resolved
if require_explicit:
return None, {}
first_name = next(iter(instances.keys()))
resolved = dict(instances[first_name])
if first_name != "default":
resolved.setdefault("_instance_name", first_name)
return (None if first_name == "default" else first_name), resolved
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
"""Allow providers to normalize query text and parse inline arguments.""" """Allow plugins to normalize query text and parse inline arguments."""
normalized = str(query or "").strip() normalized = str(query or "").strip()
return normalized, {} return normalized, {}
@@ -250,9 +366,9 @@ class Provider(ABC):
table_type: str = "", table_type: str = "",
table_meta: Optional[Dict[str, Any]] = None, table_meta: Optional[Dict[str, Any]] = None,
) -> Tuple[List[SearchResult], Optional[str], Optional[Dict[str, Any]]]: ) -> Tuple[List[SearchResult], Optional[str], Optional[Dict[str, Any]]]:
"""Optional hook for provider-specific result transforms. """Optional hook for plugin-specific result transforms.
Cmdlets should avoid hardcoding provider quirks. Providers can override Cmdlets should avoid hardcoding plugin quirks. Plugins can override
this to: this to:
- expand/replace result sets (e.g., artist -> albums) - expand/replace result sets (e.g., artist -> albums)
- override the table type - override the table type
@@ -282,7 +398,7 @@ class Provider(ABC):
**kwargs: Any, **kwargs: Any,
) -> List[SearchResult]: ) -> List[SearchResult]:
"""Search for items matching the query.""" """Search for items matching the query."""
raise NotImplementedError(f"Provider '{self.name}' does not support search") raise NotImplementedError(f"Plugin '{self.name}' does not support search")
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
"""Download an item from a search result.""" """Download an item from a search result."""
@@ -355,7 +471,7 @@ class Provider(ABC):
} }
def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]: def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]:
"""Optional provider override to parse and act on URLs.""" """Optional plugin override to parse and act on URLs."""
_ = url _ = url
_ = output_dir _ = output_dir
@@ -428,24 +544,24 @@ class Provider(ABC):
return "" return ""
def config_actions(self) -> List[Dict[str, Any]]: def config_actions(self) -> List[Dict[str, Any]]:
"""Optional actions exposed in the config editor for this provider.""" """Optional actions exposed in the config editor for this plugin."""
return [] return []
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]: def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
"""Execute a provider-owned config action from the config editor.""" """Execute a plugin-owned config action from the config editor."""
return { return {
"ok": False, "ok": False,
"message": f"Provider '{self.name}' does not support config action '{action_id}'.", "message": f"Plugin '{self.name}' does not support config action '{action_id}'.",
} }
def upload(self, file_path: str, **kwargs: Any) -> str: def upload(self, file_path: str, **kwargs: Any) -> str:
"""Upload a file and return a URL or identifier.""" """Upload a file and return a URL or identifier."""
raise NotImplementedError(f"Provider '{self.name}' does not support upload") raise NotImplementedError(f"Plugin '{self.name}' does not support upload")
def validate(self) -> bool: def validate(self) -> bool:
"""Check if provider is available and properly configured.""" """Check if the plugin is available and properly configured."""
return True return True
@@ -459,7 +575,7 @@ class Provider(ABC):
) -> bool: ) -> bool:
"""Optional hook for handling `@N` selection semantics. """Optional hook for handling `@N` selection semantics.
The CLI can delegate selection behavior to a provider/store instead of The CLI can delegate selection behavior to a plugin/store instead of
applying the default selection filtering. applying the default selection filtering.
Return True if the selection was handled and default behavior should be skipped. Return True if the selection was handled and default behavior should be skipped.
@@ -470,6 +586,101 @@ class Provider(ABC):
_ = stage_is_last _ = stage_is_last
return False return False
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
"""Optionally render a terminal detail view for a selected plugin row."""
_selected_item, payload, _metadata = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
)
_ = table_type
_ = table_metadata
if not isinstance(payload, dict):
return False
detail_title = self.label
item_title = str(payload.get("title") or "").strip()
if item_title:
detail_title = f"{self.label}: {item_title}"
try:
from SYS.rich_display import render_item_details_panel
render_item_details_panel(payload, title=detail_title)
except Exception:
return False
try:
if hasattr(ctx, "set_last_result_items_only"):
ctx.set_last_result_items_only([payload])
except Exception:
pass
return True
def resolve_selection_detail_subject(
self,
selected_items: List[Any],
*,
stage_is_last: bool = True,
source_command: str = "",
require_media_kind: Optional[str] = None,
) -> Tuple[Optional[Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
"""Normalize a terminal `@N` selection into `(item, payload, metadata)`.
Custom plugin detail hooks can use this to share the common preconditions
for item panels instead of re-checking terminal/single-row/search-file
state in each plugin.
"""
if not stage_is_last or len(selected_items or []) != 1:
return None, None, None
normalized_source = str(source_command or "").replace("_", "-").strip().lower()
if normalized_source != "search-file":
return None, None, None
item: Any = selected_items[0]
payload: Optional[Dict[str, Any]]
if isinstance(item, dict):
payload = item
else:
payload = None
to_dict = getattr(item, "to_dict", None)
if callable(to_dict):
try:
maybe = to_dict()
except Exception:
maybe = None
if isinstance(maybe, dict):
payload = maybe
if not isinstance(payload, dict):
return item, None, None
meta: Dict[str, Any] = {}
nested = payload.get("full_metadata") or payload.get("metadata")
if isinstance(nested, dict):
meta = nested
if require_media_kind:
media_kind = str(payload.get("media_kind") or meta.get("media_kind") or "").strip().lower()
if media_kind != str(require_media_kind or "").strip().lower():
return item, None, None
return item, payload, meta
@classmethod @classmethod
def selection_auto_stage( def selection_auto_stage(
cls, cls,
@@ -478,10 +689,10 @@ class Provider(ABC):
) -> Optional[List[str]]: ) -> Optional[List[str]]:
"""Return a stage to auto-run after selecting from `table_type`. """Return a stage to auto-run after selecting from `table_type`.
This is used by the CLI to auto-insert default stages for provider tables This is used by the CLI to auto-insert default stages for plugin tables
(e.g. select a YouTube row -> auto-run download-file). (e.g. select a YouTube row -> auto-run download-file).
Providers can implement this via class attributes (TABLE_AUTO_STAGES / Plugins can implement this via class attributes (TABLE_AUTO_STAGES /
TABLE_AUTO_PREFIXES) or by overriding this method. TABLE_AUTO_PREFIXES) or by overriding this method.
""" """
t = str(table_type or "").strip().lower() t = str(table_type or "").strip().lower()
@@ -522,7 +733,7 @@ class Provider(ABC):
@classmethod @classmethod
def url_patterns(cls) -> Tuple[str, ...]: def url_patterns(cls) -> Tuple[str, ...]:
"""Return normalized URL patterns that this provider handles.""" """Return normalized URL patterns that this plugin handles."""
patterns: List[str] = [] patterns: List[str] = []
maybe_urls = getattr(cls, "URL", None) maybe_urls = getattr(cls, "URL", None)
if isinstance(maybe_urls, (list, tuple)): if isinstance(maybe_urls, (list, tuple)):
@@ -564,15 +775,3 @@ class Provider(ABC):
return tuple(prefixes) return tuple(prefixes)
class SearchProvider(Provider):
"""Compatibility alias for older code.
Prefer inheriting from Provider directly.
"""
class FileProvider(Provider):
"""Compatibility alias for older code.
Prefer inheriting from Provider directly.
"""
+90 -99
View File
@@ -22,12 +22,24 @@ from urllib.parse import urlparse
from SYS.logger import log, debug from SYS.logger import log, debug
from ProviderCore.base import FileProvider, Provider, SearchProvider, SearchResult from ProviderCore.base import Provider, SearchResult
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH") _EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
def _class_supports_method(
plugin_class: Type[Provider],
method_name: str,
base_method: Any,
) -> bool:
try:
method = getattr(plugin_class, method_name, None)
except Exception:
return False
return callable(method) and method is not base_method
def _repo_root() -> Path: def _repo_root() -> Path:
try: try:
return Path(__file__).resolve().parents[1] return Path(__file__).resolve().parents[1]
@@ -102,34 +114,34 @@ def _iter_external_plugin_entries(plugin_dir: Path) -> Iterable[Tuple[str, Path,
return tuple(out) return tuple(out)
@dataclass(frozen=True) @dataclass(frozen=True)
class ProviderInfo: class PluginInfo:
"""Metadata about a single plugin entry.""" """Metadata about a single plugin entry."""
canonical_name: str canonical_name: str
provider_class: Type[Provider] plugin_class: Type[Provider]
module: str module: str
alias_names: Tuple[str, ...] = field(default_factory=tuple) alias_names: Tuple[str, ...] = field(default_factory=tuple)
@property @property
def supports_search(self) -> bool: def supports_search(self) -> bool:
return self.provider_class.search is not Provider.search return _class_supports_method(self.plugin_class, "search", Provider.search)
@property @property
def supports_upload(self) -> bool: def supports_upload(self) -> bool:
try: try:
exposed = bool(getattr(self.provider_class, "EXPOSE_AS_FILE_PROVIDER", True)) exposed = bool(getattr(self.plugin_class, "EXPOSE_AS_FILE_PROVIDER", True))
except Exception: except Exception:
exposed = True exposed = True
return exposed and (self.provider_class.upload is not Provider.upload) return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload)
class ProviderRegistry: class PluginRegistry:
"""Handles discovery, registration, and lookup of built-in and external plugins.""" """Handles discovery, registration, and lookup of built-in and external plugins."""
def __init__(self, package_name: str) -> None: def __init__(self, package_name: str) -> None:
self.package_name = (package_name or "").strip() self.package_name = (package_name or "").strip()
self._infos: Dict[str, ProviderInfo] = {} self._infos: Dict[str, PluginInfo] = {}
self._lookup: Dict[str, ProviderInfo] = {} self._lookup: Dict[str, PluginInfo] = {}
self._modules: set[str] = set() self._modules: set[str] = set()
self._external_modules: set[str] = set() self._external_modules: set[str] = set()
self._builtin_package_dirs: Tuple[Path, ...] = () self._builtin_package_dirs: Tuple[Path, ...] = ()
@@ -174,7 +186,7 @@ class ProviderRegistry:
return str(value or "").strip().lower() return str(value or "").strip().lower()
def _candidate_names(self, def _candidate_names(self,
provider_class: Type[Provider], plugin_class: Type[Provider],
override_name: Optional[str]) -> List[str]: override_name: Optional[str]) -> List[str]:
names: List[str] = [] names: List[str] = []
seen: set[str] = set() seen: set[str] = set()
@@ -190,25 +202,25 @@ class ProviderRegistry:
if override_name: if override_name:
_add(override_name) _add(override_name)
else: else:
_add(getattr(provider_class, "PLUGIN_NAME", None)) _add(getattr(plugin_class, "PLUGIN_NAME", None))
_add(getattr(provider_class, "__name__", None)) _add(getattr(plugin_class, "__name__", None))
for alias in getattr(provider_class, "PLUGIN_ALIASES", ()) or (): for alias in getattr(plugin_class, "PLUGIN_ALIASES", ()) or ():
_add(alias) _add(alias)
return names return names
def register( def register(
self, self,
provider_class: Type[Provider], plugin_class: Type[Provider],
*, *,
override_name: Optional[str] = None, override_name: Optional[str] = None,
extra_aliases: Optional[Sequence[str]] = None, extra_aliases: Optional[Sequence[str]] = None,
module_name: Optional[str] = None, module_name: Optional[str] = None,
replace: bool = False, replace: bool = False,
) -> ProviderInfo: ) -> PluginInfo:
"""Register a plugin class with canonical and alias names.""" """Register a plugin class with canonical and alias names."""
candidates = self._candidate_names(provider_class, override_name) candidates = self._candidate_names(plugin_class, override_name)
if not candidates: if not candidates:
raise ValueError("plugin name candidates are required") raise ValueError("plugin name candidates are required")
@@ -233,10 +245,10 @@ class ProviderRegistry:
alias_seen.add(normalized) alias_seen.add(normalized)
alias_names.append(normalized) alias_names.append(normalized)
info = ProviderInfo( info = PluginInfo(
canonical_name=canonical, canonical_name=canonical,
provider_class=provider_class, plugin_class=plugin_class,
module=module_name or getattr(provider_class, "__module__", "") or "", module=module_name or getattr(plugin_class, "__module__", "") or "",
alias_names=tuple(alias_names), alias_names=tuple(alias_names),
) )
@@ -261,7 +273,7 @@ class ProviderRegistry:
continue continue
if not issubclass(candidate, Provider): if not issubclass(candidate, Provider):
continue continue
if candidate in {Provider, SearchProvider, FileProvider}: if candidate is Provider:
continue continue
if getattr(candidate, "__module__", "") != module_name: if getattr(candidate, "__module__", "") != module_name:
continue continue
@@ -311,7 +323,7 @@ class ProviderRegistry:
log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr) log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr)
def discover(self) -> None: def discover(self) -> None:
"""Import and register providers from the package.""" """Import and register plugins from the package."""
if self._discovered or not self.package_name: if self._discovered or not self.package_name:
return return
@@ -376,7 +388,7 @@ class ProviderRegistry:
self._sync_subclasses() self._sync_subclasses()
return return
def get(self, name: str) -> Optional[ProviderInfo]: def get(self, name: str) -> Optional[PluginInfo]:
if not name: if not name:
return None return None
@@ -398,7 +410,7 @@ class ProviderRegistry:
self.discover() self.discover()
return self._lookup.get(normalized) return self._lookup.get(normalized)
def iter_providers(self) -> Iterable[ProviderInfo]: def iter_plugins(self) -> Iterable[PluginInfo]:
self.discover() self.discover()
return tuple(self._infos.values()) return tuple(self._infos.values())
@@ -406,12 +418,9 @@ class ProviderRegistry:
return self.get(name) is not None return self.get(name) is not None
def _sync_subclasses(self) -> None: def _sync_subclasses(self) -> None:
"""Walk all Provider subclasses in memory and register them.""" """Walk all plugin subclasses in memory and register them."""
def _walk(cls: Type[Provider]) -> None: def _walk(cls: Type[Provider]) -> None:
for sub in cls.__subclasses__(): for sub in cls.__subclasses__():
if sub in {SearchProvider, FileProvider}:
_walk(sub)
continue
try: try:
self.register(sub) self.register(sub)
except Exception: except Exception:
@@ -419,16 +428,14 @@ class ProviderRegistry:
_walk(sub) _walk(sub)
_walk(Provider) _walk(Provider)
REGISTRY = ProviderRegistry("plugins") REGISTRY = PluginRegistry("plugins")
PLUGIN_REGISTRY = REGISTRY PLUGIN_REGISTRY = REGISTRY
PluginInfo = ProviderInfo
PluginRegistry = ProviderRegistry
@lru_cache(maxsize=512) @lru_cache(maxsize=512)
def _provider_url_patterns(provider_class: Type[Provider]) -> Sequence[str]: def _plugin_url_patterns(plugin_class: Type[Provider]) -> Sequence[str]:
try: try:
return list(provider_class.url_patterns()) return list(plugin_class.url_patterns())
except Exception: except Exception:
return [] return []
@@ -440,7 +447,7 @@ def register_plugin(
aliases: Optional[Sequence[str]] = None, aliases: Optional[Sequence[str]] = None,
module_name: Optional[str] = None, module_name: Optional[str] = None,
replace: bool = False, replace: bool = False,
) -> ProviderInfo: ) -> PluginInfo:
return REGISTRY.register( return REGISTRY.register(
plugin_class, plugin_class,
override_name=name, override_name=name,
@@ -454,7 +461,7 @@ def get_plugin_class(name: str) -> Optional[Type[Provider]]:
info = REGISTRY.get(name) info = REGISTRY.get(name)
if info is None: if info is None:
return None return None
return info.provider_class return info.plugin_class
def selection_auto_stage_for_table( def selection_auto_stage_for_table(
@@ -481,7 +488,7 @@ def is_known_plugin_name(name: str) -> bool:
def _supports_search(provider: Provider) -> bool: def _supports_search(provider: Provider) -> bool:
return provider.__class__.search is not Provider.search return _class_supports_method(provider.__class__, "search", Provider.search)
def _supports_upload(provider: Provider) -> bool: def _supports_upload(provider: Provider) -> bool:
@@ -489,7 +496,25 @@ def _supports_upload(provider: Provider) -> bool:
exposed = bool(getattr(provider.__class__, "EXPOSE_AS_FILE_PROVIDER", True)) exposed = bool(getattr(provider.__class__, "EXPOSE_AS_FILE_PROVIDER", True))
except Exception: except Exception:
exposed = True exposed = True
return exposed and (provider.__class__.upload is not Provider.upload) return exposed and _class_supports_method(provider.__class__, "upload", Provider.upload)
def _supports_capability(provider: Provider, capability: str) -> bool:
capability_key = str(capability or "").strip().lower()
if capability_key == "search":
return _supports_search(provider)
if capability_key in {"upload", "file", "file-provider"}:
return _supports_upload(provider)
return False
def _info_supports_capability(info: PluginInfo, capability: str) -> bool:
capability_key = str(capability or "").strip().lower()
if capability_key == "search":
return bool(info.supports_search)
if capability_key in {"upload", "file", "file-provider"}:
return bool(info.supports_upload)
return False
def _normalize_choice_entry(entry: Any) -> Optional[Dict[str, Any]]: def _normalize_choice_entry(entry: Any) -> Optional[Dict[str, Any]]:
@@ -555,7 +580,7 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
return None return None
try: try:
plugin = info.provider_class(config) plugin = info.plugin_class(config)
if not plugin.validate(): if not plugin.validate():
debug(f"[plugin] Plugin '{name}' is not available") debug(f"[plugin] Plugin '{name}' is not available")
return None return None
@@ -567,78 +592,50 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {} availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers(): for info in REGISTRY.iter_plugins():
try: try:
plugin = info.provider_class(config) plugin = info.plugin_class(config)
availability[info.canonical_name] = plugin.validate() availability[info.canonical_name] = plugin.validate()
except Exception: except Exception:
availability[info.canonical_name] = False availability[info.canonical_name] = False
return availability return availability
def get_search_plugin(name: str, def get_plugin_with_capability(
config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]: name: str,
capability: str,
config: Optional[Dict[str, Any]] = None,
) -> Optional[Provider]:
plugin = get_plugin(name, config) plugin = get_plugin(name, config)
if plugin is None: if plugin is None:
return None return None
if not _supports_search(plugin): if not _supports_capability(plugin, capability):
debug(f"[plugin] Plugin '{name}' does not support search") debug(f"[plugin] Plugin '{name}' does not support capability '{capability}'")
return None return None
return plugin # type: ignore[return-value] return plugin
def list_search_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]: def list_plugins_with_capability(
capability: str,
config: Optional[Dict[str, Any]] = None,
) -> Dict[str, bool]:
availability: Dict[str, bool] = {} availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers(): for info in REGISTRY.iter_plugins():
try: try:
plugin = info.provider_class(config) plugin = info.plugin_class(config)
availability[info.canonical_name] = bool( availability[info.canonical_name] = bool(
plugin.validate() and info.supports_search plugin.validate() and _supports_capability(plugin, capability)
) )
except Exception: except Exception:
availability[info.canonical_name] = False availability[info.canonical_name] = False
return availability return availability
def list_search_plugin_names() -> List[str]: def list_plugin_names_with_capability(capability: str) -> List[str]:
"""Return registered search-provider names without instantiating plugins."""
return sorted( return sorted(
info.canonical_name info.canonical_name
for info in REGISTRY.iter_providers() for info in REGISTRY.iter_plugins()
if info.supports_search if _info_supports_capability(info, capability)
)
def get_upload_plugin(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[FileProvider]:
plugin = get_plugin(name, config)
if plugin is None:
return None
if not _supports_upload(plugin):
debug(f"[plugin] Plugin '{name}' does not support upload")
return None
return plugin # type: ignore[return-value]
def list_upload_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
try:
plugin = info.provider_class(config)
availability[info.canonical_name] = bool(
plugin.validate() and info.supports_upload
)
except Exception:
availability[info.canonical_name] = False
return availability
def list_upload_plugin_names() -> List[str]:
"""Return registered upload-provider names without instantiating plugins."""
return sorted(
info.canonical_name
for info in REGISTRY.iter_providers()
if info.supports_upload
) )
@@ -677,8 +674,8 @@ def match_plugin_name_for_url(url: str) -> Optional[str]:
return "openlibrary" if REGISTRY.has_name("openlibrary") else None return "openlibrary" if REGISTRY.has_name("openlibrary") else None
return "internetarchive" if REGISTRY.has_name("internetarchive") else None return "internetarchive" if REGISTRY.has_name("internetarchive") else None
for info in REGISTRY.iter_providers(): for info in REGISTRY.iter_plugins():
domains = _provider_url_patterns(info.provider_class) domains = _plugin_url_patterns(info.plugin_class)
if not domains: if not domains:
continue continue
for domain in domains: for domain in domains:
@@ -721,11 +718,9 @@ def plugin_inline_query_choices(
mapping: Dict[str, List[Dict[str, Any]]] = {} mapping: Dict[str, List[Dict[str, Any]]] = {}
info = REGISTRY.get(pname) info = REGISTRY.get(pname)
if info is not None: if info is not None:
mapping = _collect_inline_choice_mapping(info.provider_class) mapping = _collect_inline_choice_mapping(info.plugin_class)
if not mapping: if not mapping:
plugin = get_search_plugin(pname, config)
if plugin is None:
plugin = get_plugin(pname, config) plugin = get_plugin(pname, config)
if plugin is None: if plugin is None:
return [] return []
@@ -770,9 +765,9 @@ def get_plugin_for_url(url: str,
def list_selection_url_prefixes() -> List[str]: def list_selection_url_prefixes() -> List[str]:
prefixes: List[str] = [] prefixes: List[str] = []
seen: set[str] = set() seen: set[str] = set()
for info in REGISTRY.iter_providers(): for info in REGISTRY.iter_plugins():
try: try:
values = info.provider_class.selection_url_prefixes() values = info.plugin_class.selection_url_prefixes()
except Exception: except Exception:
values = () values = ()
for value in values or (): for value in values or ():
@@ -842,21 +837,17 @@ def resolve_inline_filters(
__all__ = [ __all__ = [
"ProviderInfo",
"PluginInfo", "PluginInfo",
"Provider", "Provider",
"SearchProvider",
"FileProvider",
"SearchResult", "SearchResult",
"PluginRegistry", "PluginRegistry",
"PLUGIN_REGISTRY", "PLUGIN_REGISTRY",
"register_plugin", "register_plugin",
"get_plugin", "get_plugin",
"list_plugins", "list_plugins",
"get_search_plugin", "get_plugin_with_capability",
"list_search_plugins", "list_plugins_with_capability",
"get_upload_plugin", "list_plugin_names_with_capability",
"list_upload_plugins",
"match_plugin_name_for_url", "match_plugin_name_for_url",
"get_plugin_for_url", "get_plugin_for_url",
"list_selection_url_prefixes", "list_selection_url_prefixes",
+81 -3
View File
@@ -3,8 +3,41 @@ from __future__ import annotations
from typing import Any, Iterable, Optional, Sequence from typing import Any, Iterable, Optional, Sequence
_ACRONYM_LABELS = {
"id": "ID",
"ids": "IDs",
"url": "URL",
"urls": "URLs",
"api": "API",
"http": "HTTP",
"https": "HTTPS",
"ftp": "FTP",
"ftps": "FTPS",
"scp": "SCP",
"ssh": "SSH",
"ip": "IP",
"mpv": "MPV",
}
def _labelize_key(key: str) -> str: def _labelize_key(key: str) -> str:
return str(key or "").replace("_", " ").title() parts = [part for part in str(key or "").replace("_", " ").split() if part]
normalized: list[str] = []
for part in parts:
lowered = part.lower()
normalized.append(_ACRONYM_LABELS.get(lowered, part.title()))
return " ".join(normalized)
def _has_display_value(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
text = value.strip()
return bool(text and text.lower() not in {"<null>", "null", "none"})
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return any(_has_display_value(item) for item in value)
return True
def _normalize_tags_value(tags: Any) -> Optional[str]: def _normalize_tags_value(tags: Any) -> Optional[str]:
@@ -45,7 +78,7 @@ def prepare_detail_metadata(
if str(key).startswith("_") or key in {"selection_action", "selection_args"}: if str(key).startswith("_") or key in {"selection_action", "selection_args"}:
continue continue
label = _labelize_key(str(key)) label = _labelize_key(str(key))
if label not in metadata and value is not None: if label not in metadata and _has_display_value(value):
metadata[label] = value metadata[label] = value
if title: if title:
@@ -62,7 +95,7 @@ def prepare_detail_metadata(
metadata["Tags"] = tags_text metadata["Tags"] = tags_text
for key, value in (extra_fields or {}).items(): for key, value in (extra_fields or {}).items():
if value is not None: if _has_display_value(value):
metadata[str(key)] = value metadata[str(key)] = value
return metadata return metadata
@@ -77,6 +110,7 @@ def create_detail_view(
init_command: Optional[tuple[str, Sequence[str]]] = None, init_command: Optional[tuple[str, Sequence[str]]] = None,
max_columns: Optional[int] = None, max_columns: Optional[int] = None,
exclude_tags: bool = False, exclude_tags: bool = False,
detail_order: Optional[Sequence[str]] = None,
value_case: Optional[str] = "preserve", value_case: Optional[str] = "preserve",
perseverance: bool = True, perseverance: bool = True,
) -> Any: ) -> Any:
@@ -87,6 +121,8 @@ def create_detail_view(
kwargs["max_columns"] = max_columns kwargs["max_columns"] = max_columns
if exclude_tags: if exclude_tags:
kwargs["exclude_tags"] = True kwargs["exclude_tags"] = True
if detail_order is not None:
kwargs["detail_order"] = list(detail_order)
table = ItemDetailView(title, **kwargs) table = ItemDetailView(title, **kwargs)
if table_name: if table_name:
@@ -102,3 +138,45 @@ def create_detail_view(
name, args = init_command name, args = init_command
table = table.init_command(name, list(args)) table = table.init_command(name, list(args))
return table return table
def render_selection_detail_view(
*,
ctx: Any,
item: Any,
title: str,
metadata: dict[str, Any],
table_name: Optional[str] = None,
detail_order: Optional[Sequence[str]] = None,
value_case: Optional[str] = "preserve",
exclude_tags: bool = False,
) -> bool:
from SYS.rich_display import stdout_console
detail_view = create_detail_view(
title,
metadata,
table_name=table_name,
detail_order=detail_order,
value_case=value_case,
exclude_tags=exclude_tags,
)
payload = item.to_dict() if hasattr(item, "to_dict") else item
try:
if hasattr(ctx, "set_last_result_items_only") and isinstance(payload, dict):
ctx.set_last_result_items_only([payload])
except Exception:
pass
try:
try:
detail_view.title = ""
detail_view.header_lines = []
except Exception:
pass
stdout_console().print()
stdout_console().print(detail_view.to_rich())
except Exception:
return False
return True
+34 -4
View File
@@ -1478,13 +1478,18 @@ class PipelineExecutor:
config: Any, config: Any,
selected_items: list, selected_items: list,
*, *,
stage_is_last: bool stage_is_last: bool,
source_command: Any = None,
prefer_detail_fallback: bool = False,
) -> bool: ) -> bool:
if not stage_is_last: if not stage_is_last:
return False return False
candidates: list[str] = [] candidates: list[str] = []
seen: set[str] = set() seen: set[str] = set()
current_table = None
table_meta = None
table_type = ""
def _add(value) -> None: def _add(value) -> None:
try: try:
@@ -1504,6 +1509,8 @@ class PipelineExecutor:
table if current_table and hasattr(current_table, table if current_table and hasattr(current_table,
"table") else None "table") else None
) )
if current_table and hasattr(current_table, "table"):
table_type = str(getattr(current_table, "table", "") or "").strip()
# Prefer an explicit plugin hint from table metadata when available. # Prefer an explicit plugin hint from table metadata when available.
# This keeps @N selectors working even when row payloads don't carry a # This keeps @N selectors working even when row payloads don't carry a
@@ -1516,6 +1523,7 @@ class PipelineExecutor:
) )
except Exception: except Exception:
meta = None meta = None
table_meta = meta if isinstance(meta, dict) else None
if isinstance(meta, dict): if isinstance(meta, dict):
_add(meta.get("plugin")) _add(meta.get("plugin"))
_add(meta.get("provider")) _add(meta.get("provider"))
@@ -1585,6 +1593,26 @@ class PipelineExecutor:
if handled: if handled:
return True return True
if prefer_detail_fallback:
detail_renderer = getattr(provider, "show_selection_details", None)
if callable(detail_renderer):
try:
detail_handled = bool(
detail_renderer(
selected_items,
ctx=ctx,
stage_is_last=True,
source_command=str(source_command or ""),
table_type=table_type,
table_metadata=table_meta,
)
)
except Exception as exc:
logger.exception("%s detail fallback failed during selection: %s", key, exc)
return True
if detail_handled:
return True
@staticmethod @staticmethod
def _maybe_expand_plugin_selection( def _maybe_expand_plugin_selection(
selected_items: List[Any], selected_items: List[Any],
@@ -2183,7 +2211,9 @@ class PipelineExecutor:
ctx, ctx,
config, config,
filtered, filtered,
stage_is_last=(not stages)): stage_is_last=(not stages),
source_command=source_cmd,
prefer_detail_fallback=bool(prefer_row_action and not stages and len(selection_indices) == 1)):
return False, None return False, None
from SYS.pipe_object import coerce_to_pipe_object from SYS.pipe_object import coerce_to_pipe_object
@@ -2204,7 +2234,7 @@ class PipelineExecutor:
except Exception: except Exception:
logger.exception("Failed to record Applied @N selection log step (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) logger.exception("Failed to record Applied @N selection log step (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None))
# Auto-insert downloader stages for provider tables. # Auto-insert downloader stages for plugin tables.
try: try:
current_table = ctx.get_current_stage_table() current_table = ctx.get_current_stage_table()
if current_table is None and hasattr(ctx, "get_display_table"): if current_table is None and hasattr(ctx, "get_display_table"):
@@ -2360,7 +2390,7 @@ class PipelineExecutor:
# Multi-selection fallback: if any selected row declares a # Multi-selection fallback: if any selected row declares a
# download-file action, insert a generic download-file stage. # download-file action, insert a generic download-file stage.
# This keeps provider-specific behavior in provider metadata. # This keeps plugin-specific behavior in plugin metadata.
if (not inserted_provider_download) and len(selection_indices) > 1: if (not inserted_provider_download) and len(selection_indices) > 1:
try: try:
has_download_row_action = False has_download_row_action = False
-12
View File
@@ -68,10 +68,6 @@ def get_plugin_schema(plugin_name: str) -> List[ConfigField]:
return _call_schema(plugin_class, f"plugin '{plugin_name}'") return _call_schema(plugin_class, f"plugin '{plugin_name}'")
def get_provider_schema(provider_name: str) -> List[ConfigField]:
return get_plugin_schema(provider_name)
def get_tool_schema(tool_name: str) -> List[ConfigField]: def get_tool_schema(tool_name: str) -> List[ConfigField]:
tool_name = str(tool_name or "").strip() tool_name = str(tool_name or "").strip()
if not tool_name: if not tool_name:
@@ -149,10 +145,6 @@ def build_default_plugin_config(plugin_name: str) -> Dict[str, Any]:
return config return config
def build_default_provider_config(provider_name: str) -> Dict[str, Any]:
return build_default_plugin_config(provider_name)
def build_default_tool_config(tool_name: str) -> Dict[str, Any]: def build_default_tool_config(tool_name: str) -> Dict[str, Any]:
config: Dict[str, Any] = {} config: Dict[str, Any] = {}
for field in get_tool_schema(tool_name): for field in get_tool_schema(tool_name):
@@ -215,10 +207,6 @@ def get_configurable_plugin_types() -> List[str]:
return sorted(set(options)) return sorted(set(options))
def get_configurable_provider_types() -> List[str]:
return get_configurable_plugin_types()
def get_configurable_tool_types() -> List[str]: def get_configurable_tool_types() -> List[str]:
options: List[str] = [] options: List[str] = []
try: try:
+59 -12
View File
@@ -724,7 +724,7 @@ class Table:
"""Table type (e.g., 'youtube', 'soulseek') for context-aware selection logic.""" """Table type (e.g., 'youtube', 'soulseek') for context-aware selection logic."""
self.table_metadata: Dict[str, Any] = {} self.table_metadata: Dict[str, Any] = {}
"""Optional provider/table metadata (e.g., provider name, view).""" """Optional plugin/table metadata (e.g., plugin name, view)."""
self.value_case: str = "preserve" self.value_case: str = "preserve"
"""Display-only value casing: 'lower', 'upper', or 'preserve' (default).""" """Display-only value casing: 'lower', 'upper', or 'preserve' (default)."""
@@ -754,12 +754,12 @@ class Table:
return self return self
def set_table_metadata(self, metadata: Optional[Dict[str, Any]]) -> "Table": def set_table_metadata(self, metadata: Optional[Dict[str, Any]]) -> "Table":
"""Attach provider/table metadata for downstream selection logic.""" """Attach plugin/table metadata for downstream selection logic."""
self.table_metadata = dict(metadata or {}) self.table_metadata = dict(metadata or {})
return self return self
def get_table_metadata(self) -> Dict[str, Any]: def get_table_metadata(self) -> Dict[str, Any]:
"""Return attached provider/table metadata (copy to avoid mutation).""" """Return attached plugin/table metadata (copy to avoid mutation)."""
try: try:
return dict(self.table_metadata) return dict(self.table_metadata)
except Exception: except Exception:
@@ -2224,6 +2224,34 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
out = {} out = {}
def _merge_columns(columns_value: Any) -> None:
if not isinstance(columns_value, (list, tuple)):
return
for column in columns_value:
label = None
value = None
if isinstance(column, (list, tuple)) and len(column) >= 2:
label, value = column[0], column[1]
elif isinstance(column, dict):
label = column.get("name") or column.get("label") or column.get("key")
value = column.get("value")
else:
label = getattr(column, "name", None)
value = getattr(column, "value", None)
label_text = str(label or "").strip()
if not label_text or value is None:
continue
value_text = str(value).strip()
if not value_text:
continue
normalized = label_text.lower()
if any(str(existing or "").strip().lower() == normalized for existing in out):
continue
out[label_text] = value_text
# Handle ResultModel specifically for better detail display # Handle ResultModel specifically for better detail display
if ResultModel is not None and isinstance(item, ResultModel): if ResultModel is not None and isinstance(item, ResultModel):
if item.title: out["Title"] = item.title if item.title: out["Title"] = item.title
@@ -2231,6 +2259,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
if item.ext: out["Ext"] = item.ext if item.ext: out["Ext"] = item.ext
if item.size_bytes: out["Size"] = format_mb(item.size_bytes) if item.size_bytes: out["Size"] = format_mb(item.size_bytes)
if item.source: out["Store"] = item.source if item.source: out["Store"] = item.source
_merge_columns(getattr(item, "columns", None))
# Merge internal metadata dict # Merge internal metadata dict
if item.metadata: if item.metadata:
@@ -2256,6 +2285,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
# Fallback to existing extraction logic for legacy objects/dicts # Fallback to existing extraction logic for legacy objects/dicts
# Convert once and reuse throughout to avoid repeated _as_dict() calls # Convert once and reuse throughout to avoid repeated _as_dict() calls
data = _as_dict(item) or {} data = _as_dict(item) or {}
_merge_columns(data.get("columns"))
# Use existing extractors from match-standard result table columns # Use existing extractors from match-standard result table columns
title = extract_title_value(item) title = extract_title_value(item)
@@ -2350,12 +2380,14 @@ class ItemDetailView(Table):
item_metadata: Optional[Dict[str, Any]] = None, item_metadata: Optional[Dict[str, Any]] = None,
detail_title: Optional[str] = None, detail_title: Optional[str] = None,
exclude_tags: bool = False, exclude_tags: bool = False,
detail_order: Optional[List[str]] = None,
**kwargs **kwargs
): ):
super().__init__(title, **kwargs) super().__init__(title, **kwargs)
self.item_metadata = item_metadata or {} self.item_metadata = item_metadata or {}
self.detail_title = detail_title self.detail_title = detail_title
self.exclude_tags = exclude_tags self.exclude_tags = exclude_tags
self.detail_order = [str(value) for value in (detail_order or []) if str(value or "").strip()]
def to_rich(self): def to_rich(self):
"""Render the item details panel above the standard results table.""" """Render the item details panel above the standard results table."""
@@ -2406,8 +2438,26 @@ class ItemDetailView(Table):
return Group(*renderables) return Group(*renderables)
# Canonical display order for metadata def _has_renderable_value(value: Any) -> bool:
order = ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"] if value is None:
return False
if isinstance(value, str):
text = value.strip()
return bool(text and text.lower() not in {"<null>", "null", "none"})
if isinstance(value, (list, tuple, set)):
return any(_has_renderable_value(item) for item in value)
return True
# Canonical display order for metadata; plugin-specific detail views can
# prepend a preferred order without needing to reimplement rendering.
order: List[str] = []
seen_order: set[str] = set()
for key in list(self.detail_order or []) + ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"]:
normalized = str(key or "").strip().lower()
if not normalized or normalized in seen_order:
continue
seen_order.add(normalized)
order.append(str(key))
has_details = False has_details = False
# Add ordered items first # Add ordered items first
@@ -2431,19 +2481,16 @@ class ItemDetailView(Table):
else: else:
val = "\n".join([f"[dim]→[/dim] {r}" for r in val]) val = "\n".join([f"[dim]→[/dim] {r}" for r in val])
if val is not None and val != "": if _has_renderable_value(val):
details_table.add_row(f"{key}:", str(val)) details_table.add_row(f"{key}:", str(val))
has_details = True has_details = True
elif key in ["Url", "Relations", "Ext"]:
# Show <null> for these important identifier fields if blank
details_table.add_row(f"{key}:", "[dim]<null>[/dim]")
has_details = True
# Add any remaining metadata not in the canonical list # Add any remaining metadata not in the canonical list
ordered_keys = {x.lower() for x in order}
for k, v in self.item_metadata.items(): for k, v in self.item_metadata.items():
k_norm = k.lower() k_norm = k.lower()
if k_norm not in [x.lower() for x in order] and v and k_norm not in ["tags", "tag"]: if k_norm not in ordered_keys and _has_renderable_value(v) and k_norm not in ["tags", "tag"]:
label = k.capitalize() if len(k) > 1 else k.upper() label = str(k or "")
details_table.add_row(f"{label}:", str(v)) details_table.add_row(f"{label}:", str(v))
has_details = True has_details = True
+12 -12
View File
@@ -77,26 +77,26 @@ def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]:
_STDERR_CONSOLE = previous_stderr _STDERR_CONSOLE = previous_stderr
def show_provider_config_panel( def show_plugin_config_panel(
provider_names: str | List[str], plugin_names: str | List[str],
) -> None: ) -> None:
"""Show a Rich panel explaining how to configure providers.""" """Show a Rich panel explaining how to configure plugins."""
from rich.table import Table as RichTable from rich.table import Table as RichTable
from rich.console import Group from rich.console import Group
if isinstance(provider_names, str): if isinstance(plugin_names, str):
providers = [p.strip() for p in provider_names.split(",")] plugins = [p.strip() for p in plugin_names.split(",")]
else: else:
providers = provider_names plugins = plugin_names
table = RichTable.grid(padding=(0, 1)) table = RichTable.grid(padding=(0, 1))
table.add_column(style="bold red") table.add_column(style="bold red")
for provider in providers: for plugin in plugins:
table.add_row(f"{provider}") table.add_row(f"{plugin}")
group = Group( group = Group(
Text("The following providers are not configured and cannot be used:\n"), Text("The following plugins are not configured and cannot be used:\n"),
table, table,
Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.") Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.")
) )
@@ -147,17 +147,17 @@ def show_store_config_panel(
stdout_console().print(panel) stdout_console().print(panel)
def show_available_providers_panel(provider_names: List[str]) -> None: def show_available_plugins_panel(plugin_names: List[str]) -> None:
"""Show a Rich panel listing available/configured plugins.""" """Show a Rich panel listing available/configured plugins."""
from rich.columns import Columns from rich.columns import Columns
from rich.console import Group from rich.console import Group
if not provider_names: if not plugin_names:
return return
# Use Columns to display them efficiently in the panel # Use Columns to display them efficiently in the panel
cols = Columns( cols = Columns(
[f"[bold green] \u2713 [/bold green]{p}" for p in sorted(provider_names)], [f"[bold green] \u2713 [/bold green]{p}" for p in sorted(plugin_names)],
equal=True, equal=True,
column_first=True, column_first=True,
expand=True expand=True
+18 -17
View File
@@ -4,6 +4,7 @@ import re
import sys import sys
import tempfile import tempfile
import shutil import shutil
from collections.abc import Mapping, Sequence as SequenceABC
from collections import deque from collections import deque
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
@@ -169,7 +170,7 @@ class HydrusNetwork(Store):
api_key: Hydrus Client API access key api_key: Hydrus Client API access key
url: Hydrus client URL (e.g., 'http://192.168.1.230:45869') url: Hydrus client URL (e.g., 'http://192.168.1.230:45869')
""" """
from API.HydrusNetwork import HydrusNetwork as HydrusClient from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient
if instance_name is None and NAME is not None: if instance_name is None and NAME is not None:
instance_name = str(NAME) instance_name = str(NAME)
@@ -713,7 +714,7 @@ class HydrusNetwork(Store):
"""Best-effort URL search by scanning Hydrus metadata with include_file_url=True.""" """Best-effort URL search by scanning Hydrus metadata with include_file_url=True."""
try: try:
from API.HydrusNetwork import _generate_hydrus_url_variants from plugins.hydrusnetwork.api import _generate_hydrus_url_variants
except Exception: except Exception:
_generate_hydrus_url_variants = None # type: ignore[assignment] _generate_hydrus_url_variants = None # type: ignore[assignment]
@@ -808,7 +809,7 @@ class HydrusNetwork(Store):
self._has_url_predicate = True self._has_url_predicate = True
except Exception as exc: except Exception as exc:
try: try:
from API.HydrusNetwork import HydrusRequestError from plugins.hydrusnetwork.api import HydrusRequestError
if isinstance(exc, HydrusRequestError) and getattr(exc, "status", None) == 400: if isinstance(exc, HydrusRequestError) and getattr(exc, "status", None) == 400:
self._has_url_predicate = False self._has_url_predicate = False
@@ -1844,7 +1845,7 @@ class HydrusNetwork(Store):
extracted_tags = self._extract_tags_from_hydrus_meta( extracted_tags = self._extract_tags_from_hydrus_meta(
meta, meta,
service_key=None, service_key=None,
service_name="my tags", service_name=None,
) )
for raw_tag in extracted_tags: for raw_tag in extracted_tags:
tag_text = str(raw_tag or "").strip() tag_text = str(raw_tag or "").strip()
@@ -2201,7 +2202,7 @@ class HydrusNetwork(Store):
return [] return []
try: try:
from API.HydrusNetwork import HydrusRequestSpec, _generate_hydrus_url_variants from plugins.hydrusnetwork.api import HydrusRequestSpec, _generate_hydrus_url_variants
except Exception: except Exception:
return [] return []
@@ -2339,7 +2340,7 @@ class HydrusNetwork(Store):
try: try:
return client.get_url_info(u) # type: ignore[attr-defined] return client.get_url_info(u) # type: ignore[attr-defined]
except Exception: except Exception:
from API.HydrusNetwork import HydrusRequestSpec from plugins.hydrusnetwork.api import HydrusRequestSpec
spec = HydrusRequestSpec( spec = HydrusRequestSpec(
method="GET", method="GET",
@@ -2563,7 +2564,7 @@ class HydrusNetwork(Store):
meta: Dict[str, meta: Dict[str,
Any], Any],
service_key: Optional[str], service_key: Optional[str],
service_name: str service_name: Optional[str]
) -> List[str]: ) -> List[str]:
"""Extract current tags from Hydrus metadata dict. """Extract current tags from Hydrus metadata dict.
@@ -2571,7 +2572,7 @@ class HydrusNetwork(Store):
Falls back to storage_tags status '0' (current). Falls back to storage_tags status '0' (current).
""" """
tags_payload = meta.get("tags") tags_payload = meta.get("tags")
if not isinstance(tags_payload, dict): if not isinstance(tags_payload, Mapping):
return [] return []
desired_service_name = str(service_name or "").strip().lower() desired_service_name = str(service_name or "").strip().lower()
@@ -2593,20 +2594,20 @@ class HydrusNetwork(Store):
out.append(cleaned) out.append(cleaned)
def _collect_current(container: Any, out: List[str]) -> None: def _collect_current(container: Any, out: List[str]) -> None:
if isinstance(container, list): if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)):
for tag in container: for tag in container:
_append_tag(out, tag) _append_tag(out, tag)
return return
if isinstance(container, dict): if isinstance(container, Mapping):
current = container.get("0") current = container.get("0")
if current is None: if current is None:
current = container.get(0) current = container.get(0)
if isinstance(current, list): if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)):
for tag in current: for tag in current:
_append_tag(out, tag) _append_tag(out, tag)
def _collect_service_data(service_data: Any, out: List[str]) -> None: def _collect_service_data(service_data: Any, out: List[str]) -> None:
if not isinstance(service_data, dict): if not isinstance(service_data, Mapping):
return return
display = ( display = (
@@ -2630,7 +2631,7 @@ class HydrusNetwork(Store):
if not collected and desired_service_name: if not collected and desired_service_name:
for maybe_service in tags_payload.values(): for maybe_service in tags_payload.values():
if not isinstance(maybe_service, dict): if not isinstance(maybe_service, Mapping):
continue continue
svc_name = str( svc_name = str(
maybe_service.get("service_name") maybe_service.get("service_name")
@@ -2642,11 +2643,11 @@ class HydrusNetwork(Store):
names_map = tags_payload.get("service_keys_to_names") names_map = tags_payload.get("service_keys_to_names")
statuses_map = tags_payload.get("service_keys_to_statuses_to_tags") statuses_map = tags_payload.get("service_keys_to_statuses_to_tags")
if isinstance(statuses_map, dict): if isinstance(statuses_map, Mapping):
keys_to_collect: List[str] = [] keys_to_collect: List[str] = []
if desired_service_key: if desired_service_key:
keys_to_collect.append(desired_service_key) keys_to_collect.append(desired_service_key)
if desired_service_name and isinstance(names_map, dict): if desired_service_name and isinstance(names_map, Mapping):
for raw_key, raw_name in names_map.items(): for raw_key, raw_name in names_map.items():
if str(raw_name or "").strip().lower() == desired_service_name: if str(raw_name or "").strip().lower() == desired_service_name:
keys_to_collect.append(str(raw_key)) keys_to_collect.append(str(raw_key))
@@ -2663,7 +2664,7 @@ class HydrusNetwork(Store):
_collect_service_data(maybe_service, collected) _collect_service_data(maybe_service, collected)
top_level_tags = meta.get("tags_flat") top_level_tags = meta.get("tags_flat")
if isinstance(top_level_tags, list): if isinstance(top_level_tags, SequenceABC) and not isinstance(top_level_tags, (str, bytes, bytearray, Mapping)):
_collect_current(top_level_tags, collected) _collect_current(top_level_tags, collected)
deduped: List[str] = [] deduped: List[str] = []
@@ -2682,7 +2683,7 @@ class HydrusNetwork(Store):
tags = HydrusNetwork._extract_tags_from_hydrus_meta( tags = HydrusNetwork._extract_tags_from_hydrus_meta(
meta, meta,
service_key=None, service_key=None,
service_name="my tags", service_name=None,
) )
normalized_tags: List[str] = [] normalized_tags: List[str] = []
+28 -28
View File
@@ -22,10 +22,10 @@ from SYS.config import (
from SYS.database import db from SYS.database import db
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.plugin_config import ( from SYS.plugin_config import (
build_default_provider_config, build_default_plugin_config,
build_default_store_config, build_default_store_config,
build_default_tool_config, build_default_tool_config,
get_configurable_provider_types, get_configurable_plugin_types,
get_configurable_store_types, get_configurable_store_types,
get_configurable_tool_types, get_configurable_tool_types,
get_global_schema, get_global_schema,
@@ -161,7 +161,7 @@ class ConfigModal(ModalScreen):
# Load config from the workspace root (parent of SYS) # Load config from the workspace root (parent of SYS)
self.config_data = load_config() self.config_data = load_config()
self.current_category = "globals" self.current_category = "globals"
self.editing_item_type = None # 'store' or 'provider' self.editing_item_type = None # 'store' or 'plugin'
self.editing_item_name = None self.editing_item_name = None
self._button_id_map = {} self._button_id_map = {}
self._provider_button_map: Dict[str, tuple[str, str]] = {} self._provider_button_map: Dict[str, tuple[str, str]] = {}
@@ -598,14 +598,14 @@ class ConfigModal(ModalScreen):
row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn"))
idx += 1 idx += 1
if item_type == "provider" and isinstance(item_name, str): if item_type == "plugin" and isinstance(item_name, str):
provider = self._instantiate_provider_for_editor(item_name, self.config_data) provider = self._instantiate_plugin_for_editor(item_name, self.config_data)
if provider is not None: if provider is not None:
provider_actions = provider.config_actions() or [] provider_actions = provider.config_actions() or []
if provider_actions: if provider_actions:
container.mount(Rule()) container.mount(Rule())
container.mount(Label(f"{provider.label} helpers", classes="config-label")) container.mount(Label(f"{provider.label} helpers", classes="config-label"))
helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip() helper_text = str(provider.config_helper_text() or "Use these helpers to validate plugin settings.").strip()
status = Static(helper_text, id="provider-status") status = Static(helper_text, id="provider-status")
container.mount(status) container.mount(status)
self._provider_status = status self._provider_status = status
@@ -626,7 +626,7 @@ class ConfigModal(ModalScreen):
) )
if ( if (
item_type == "provider" item_type == "plugin"
and isinstance(item_name, str) and isinstance(item_name, str)
and item_name.strip().lower() == "matrix" and item_name.strip().lower() == "matrix"
): ):
@@ -870,12 +870,12 @@ class ConfigModal(ModalScreen):
self.refresh_view() self.refresh_view()
elif bid in self._provider_button_map: elif bid in self._provider_button_map:
provider_name, action_id = self._provider_button_map[bid] provider_name, action_id = self._provider_button_map[bid]
self._request_provider_action(provider_name, action_id) self._request_plugin_action(provider_name, action_id)
elif bid == "add-store-btn": elif bid == "add-store-btn":
options = get_configurable_store_types() options = get_configurable_store_types()
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected) self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
elif bid == "add-provider-btn": elif bid == "add-provider-btn":
options = get_configurable_provider_types() options = get_configurable_plugin_types()
self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected) self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected)
elif bid == "add-tool-btn": elif bid == "add-tool-btn":
options = get_configurable_tool_types() or ["ytdlp"] options = get_configurable_tool_types() or ["ytdlp"]
@@ -971,46 +971,46 @@ class ConfigModal(ModalScreen):
else: else:
self.notify("Clipboard not supported in this terminal", severity="warning") self.notify("Clipboard not supported in this terminal", severity="warning")
def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]: def _instantiate_plugin_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]:
try: try:
provider_class = get_plugin_class(provider_name) plugin_class = get_plugin_class(provider_name)
except Exception: except Exception:
provider_class = None plugin_class = None
if provider_class is None: if plugin_class is None:
return None return None
try: try:
return provider_class(config_data or self.config_data) return plugin_class(config_data or self.config_data)
except Exception: except Exception:
logger.exception("Failed to instantiate provider '%s' for config helper", provider_name) logger.exception("Failed to instantiate plugin '%s' for config helper", provider_name)
return None return None
def _request_provider_action(self, provider_name: str, action_id: str) -> None: def _request_plugin_action(self, provider_name: str, action_id: str) -> None:
if self._provider_action_running: if self._provider_action_running:
return return
self._synchronize_inputs_to_config() self._synchronize_inputs_to_config()
self._provider_action_running = True self._provider_action_running = True
if self._provider_status is not None: if self._provider_status is not None:
self._provider_status.update(f"Running {action_id.replace('_', ' ')}") self._provider_status.update(f"Running {action_id.replace('_', ' ')}")
self._provider_action_background(provider_name, action_id, deepcopy(self.config_data)) self._plugin_action_background(provider_name, action_id, deepcopy(self.config_data))
@work(thread=True) @work(thread=True)
def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None: def _plugin_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None:
try: try:
provider = self._instantiate_provider_for_editor(provider_name, config_snapshot) provider = self._instantiate_plugin_for_editor(provider_name, config_snapshot)
if provider is None: if provider is None:
raise RuntimeError(f"Provider '{provider_name}' is unavailable") raise RuntimeError(f"Plugin '{provider_name}' is unavailable")
result = provider.run_config_action(action_id) result = provider.run_config_action(action_id)
if not isinstance(result, dict): if not isinstance(result, dict):
result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."} result = {"ok": False, "message": f"Plugin '{provider_name}' returned an invalid config action result."}
except Exception as exc: except Exception as exc:
result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."} result = {"ok": False, "message": str(exc) or f"Plugin action '{action_id}' failed."}
try: try:
self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result) self.app.call_from_thread(self._plugin_action_complete, provider_name, action_id, result)
except Exception: except Exception:
self._provider_action_complete(provider_name, action_id, result) self._plugin_action_complete(provider_name, action_id, result)
def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None: def _plugin_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None:
self._provider_action_running = False self._provider_action_running = False
ok = bool(result.get("ok")) ok = bool(result.get("ok"))
message = str(result.get("message") or f"Provider action '{action_id}' finished.") message = str(result.get("message") or f"Provider action '{action_id}' finished.")
@@ -1075,11 +1075,11 @@ class ConfigModal(ModalScreen):
if "provider" not in self.config_data: if "provider" not in self.config_data:
self.config_data["provider"] = {} self.config_data["provider"] = {}
# For providers, they are usually top-level entries in 'provider' dict # Plugins are configured under the top-level 'provider' dict for now.
if ptype not in self.config_data["provider"]: if ptype not in self.config_data["provider"]:
self.config_data["provider"][ptype] = build_default_provider_config(ptype) self.config_data["provider"][ptype] = build_default_plugin_config(ptype)
self.editing_item_type = "provider" self.editing_item_type = "plugin"
self.editing_item_name = ptype self.editing_item_name = ptype
self.refresh_view() self.refresh_view()
+3 -3
View File
@@ -16,7 +16,7 @@ import asyncio
sys.path.insert(0, str(Path(__file__).parent.parent)) sys.path.insert(0, str(Path(__file__).parent.parent))
from SYS.config import load_config, resolve_output_dir from SYS.config import load_config, resolve_output_dir
from SYS.result_table import Table from SYS.result_table import Table
from ProviderCore.registry import get_search_plugin from ProviderCore.registry import get_plugin_with_capability
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -174,7 +174,7 @@ class SearchModal(ModalScreen):
self.current_worker.log_step(f"Connecting to {source}...") self.current_worker.log_step(f"Connecting to {source}...")
try: try:
provider = get_search_plugin(source) provider = get_plugin_with_capability(source, "search")
if not provider: if not provider:
logger.error(f"[search-modal] Provider not available: {source}") logger.error(f"[search-modal] Provider not available: {source}")
if self.current_worker: if self.current_worker:
@@ -380,7 +380,7 @@ class SearchModal(ModalScreen):
config = load_config() config = load_config()
output_dir = resolve_output_dir(config) output_dir = resolve_output_dir(config)
provider = get_search_plugin("openlibrary", config=config) provider = get_plugin_with_capability("openlibrary", "search", config=config)
if not provider: if not provider:
logger.error("[search-modal] Provider not available: openlibrary") logger.error("[search-modal] Provider not available: openlibrary")
return return
+10 -2
View File
@@ -190,10 +190,18 @@ class SharedArgs:
name="store", name="store",
type="enum", type="enum",
choices=[], # Dynamically populated via get_store_choices() choices=[], # Dynamically populated via get_store_choices()
description="Selects store", description="Selects a storage backend",
query_key="store", query_key="store",
) )
INSTANCE = CmdletArg(
name="instance",
type="string",
description="Selects a plugin instance",
query_key="instance",
query_aliases=["store"],
)
URL = CmdletArg( URL = CmdletArg(
name="url", name="url",
type="string", type="string",
@@ -1410,7 +1418,7 @@ def fetch_hydrus_metadata(
Eliminates repeated boilerplate: client initialization, error handling, metadata extraction. Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
Args: Args:
config: Configuration object used to resolve the Hydrus provider/store config: Configuration object used to resolve the Hydrus plugin/store
hash_hex: File hash to fetch metadata for hash_hex: File hash to fetch metadata for
store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client. store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
hydrus_client: Optional explicit Hydrus client. When provided, takes precedence. hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
+42 -9
View File
@@ -15,6 +15,7 @@ from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_table_result_payload from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store from Store import Store
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
@@ -178,10 +179,11 @@ class Add_File(Cmdlet):
summary= summary=
"Ingest a local media file to a store backend, upload plugin, or local directory.", "Ingest a local media file to a store backend, upload plugin, or local directory.",
usage= usage=
"add-file (-path <filepath> | <piped>) (-storage <location> | -plugin <upload-plugin>) [-delete]", "add-file (-path <filepath> | <piped>) (-store <backend|path> | -plugin <upload-plugin>) [-instance NAME] [-delete]",
arg=[ arg=[
SharedArgs.PATH, SharedArgs.PATH,
SharedArgs.STORE, SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.URL, SharedArgs.URL,
SharedArgs.PLUGIN, SharedArgs.PLUGIN,
CmdletArg( CmdletArg(
@@ -194,7 +196,7 @@ class Add_File(Cmdlet):
], ],
detail=[ detail=[
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.", "Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
"- Storage location options (use -storage):", "- Storage location options (use -store):",
" hydrus: Upload to Hydrus database with metadata tagging", " hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory", " local: Copy file to local directory",
" <path>: Copy file to specified directory", " <path>: Copy file to specified directory",
@@ -202,9 +204,12 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting", " 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting", " file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)", " internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- In plugin mode, -store <name> is still accepted as a compatibility alias for -instance <name>.",
], ],
examples=[ examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial', 'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
], ],
exec=self.run, exec=self.run,
) )
@@ -223,9 +228,12 @@ class Add_File(Cmdlet):
path_arg = parsed.get("path") path_arg = parsed.get("path")
location = parsed.get("store") location = parsed.get("store")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url") source_url_arg = parsed.get("url")
plugin_name = parsed.get("plugin") plugin_name = parsed.get("plugin")
delete_after = parsed.get("delete", False) delete_after = parsed.get("delete", False)
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Convenience: when piping a file into add-file, allow `-path <existing dir>` # Convenience: when piping a file into add-file, allow `-path <existing dir>`
# to act as the destination export directory. # to act as the destination export directory.
@@ -412,6 +420,7 @@ class Add_File(Cmdlet):
("items", total_items), ("items", total_items),
("location", location), ("location", location),
("plugin", plugin_name), ("plugin", plugin_name),
("instance", plugin_instance),
("delete", delete_after), ("delete", delete_after),
], ],
border_style="cyan", border_style="cyan",
@@ -647,6 +656,7 @@ class Add_File(Cmdlet):
code = self._handle_plugin_upload( code = self._handle_plugin_upload(
media_path, media_path,
plugin_name, plugin_name,
plugin_instance,
pipe_obj, pipe_obj,
config, config,
delete_after_item delete_after_item
@@ -1442,9 +1452,9 @@ class Add_File(Cmdlet):
if not plugin_key: if not plugin_key:
return None, None, None return None, None, None
from ProviderCore.registry import get_search_plugin from ProviderCore.registry import get_plugin
plugin = get_search_plugin(plugin_key, config) plugin = get_plugin(plugin_key, config)
if plugin is None: if plugin is None:
return None, None, None return None, None, None
@@ -1762,6 +1772,7 @@ class Add_File(Cmdlet):
*, *,
hash_value: str, hash_value: str,
store: str, store: str,
provider: Optional[str] = None,
path: Optional[str], path: Optional[str],
tag: List[str], tag: List[str],
title: Optional[str], title: Optional[str],
@@ -1770,6 +1781,7 @@ class Add_File(Cmdlet):
) -> None: ) -> None:
pipe_obj.hash = hash_value pipe_obj.hash = hash_value
pipe_obj.store = store pipe_obj.store = store
pipe_obj.provider = provider
pipe_obj.is_temp = False pipe_obj.is_temp = False
pipe_obj.path = path pipe_obj.path = path
pipe_obj.tag = tag pipe_obj.tag = tag
@@ -2180,23 +2192,42 @@ class Add_File(Cmdlet):
def _handle_plugin_upload( def _handle_plugin_upload(
media_path: Path, media_path: Path,
plugin_name: str, plugin_name: str,
instance_name: Optional[str],
pipe_obj: models.PipeObject, pipe_obj: models.PipeObject,
config: Dict[str, config: Dict[str,
Any], Any],
delete_after: bool, delete_after: bool,
) -> int: ) -> int:
"""Handle uploading via an upload plugin (e.g. 0x0).""" """Handle uploading via an upload plugin (e.g. 0x0)."""
from ProviderCore.registry import get_upload_plugin from ProviderCore.registry import (
get_plugin_with_capability,
list_plugin_names_with_capability,
list_plugins_with_capability,
)
log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr) log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr)
try: try:
file_provider = get_upload_plugin(plugin_name, config) file_provider = get_plugin_with_capability(plugin_name, "upload", config)
if not file_provider: if not file_provider:
log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr) available_map = list_plugins_with_capability("upload", config)
known_upload_plugins = set(list_plugin_names_with_capability("upload"))
available_uploads = [name for name, enabled in available_map.items() if enabled and name in known_upload_plugins]
if str(plugin_name or "").strip().lower() in known_upload_plugins:
show_plugin_config_panel([plugin_name])
else:
log(f"Upload plugin '{plugin_name}' is not available or does not support upload", file=sys.stderr)
if available_uploads:
show_available_plugins_panel(sorted(available_uploads))
return 1 return 1
hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj) hoster_url = file_provider.upload(
str(media_path),
pipe_obj=pipe_obj,
instance=instance_name,
)
log(f"File uploaded: {hoster_url}", file=sys.stderr) log(f"File uploaded: {hoster_url}", file=sys.stderr)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None) f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
@@ -2209,6 +2240,7 @@ class Add_File(Cmdlet):
extra_updates: Dict[str, extra_updates: Dict[str,
Any] = { Any] = {
"plugin": plugin_name, "plugin": plugin_name,
"instance": instance_name,
"plugin_url": hoster_url, "plugin_url": hoster_url,
} }
if isinstance(pipe_obj.extra, dict): if isinstance(pipe_obj.extra, dict):
@@ -2222,7 +2254,8 @@ class Add_File(Cmdlet):
Add_File._update_pipe_object_destination( Add_File._update_pipe_object_destination(
pipe_obj, pipe_obj,
hash_value=f_hash or "unknown", hash_value=f_hash or "unknown",
store=plugin_name or "plugin", store="",
provider=plugin_name or None,
path=file_path, path=file_path,
tag=pipe_obj.tag, tag=pipe_obj.tag,
title=pipe_obj.title or (media_path.name if media_path else None), title=pipe_obj.title or (media_path.name if media_path else None),
+13 -13
View File
@@ -55,12 +55,13 @@ class Download_File(Cmdlet):
name="download-file", name="download-file",
summary="Download files or streaming media", summary="Download files or streaming media",
usage= usage=
"download-file <url> [-path DIR] [options] OR @N | download-file [-path DIR|DIR] [options]", "download-file <url> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]",
alias=["dl-file", alias=["dl-file",
"download-http"], "download-http"],
arg=[ arg=[
SharedArgs.URL, SharedArgs.URL,
SharedArgs.PLUGIN, SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH, SharedArgs.PATH,
SharedArgs.QUERY, SharedArgs.QUERY,
QueryArg( QueryArg(
@@ -85,6 +86,7 @@ class Download_File(Cmdlet):
], ],
detail=[ detail=[
"Download files directly via HTTP or streaming media via yt-dlp.", "Download files directly via HTTP or streaming media via yt-dlp.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.", "For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
], ],
exec=self.run, exec=self.run,
@@ -522,13 +524,13 @@ class Download_File(Cmdlet):
config: Dict[str, config: Dict[str,
Any], Any],
) -> List[Any]: ) -> List[Any]:
get_search_plugin = registry.get("get_search_plugin") get_provider = registry.get("get_plugin")
expanded_items: List[Any] = [] expanded_items: List[Any] = []
for item in piped_items: for item in piped_items:
try: try:
provider_key = self._provider_key_from_item(item) provider_key = self._provider_key_from_item(item)
provider = get_search_plugin(provider_key, config) if provider_key and get_search_plugin else None provider = get_provider(provider_key, config) if provider_key and get_provider else None
# Generic hook: If provider has expand_item(item), use it. # Generic hook: If provider has expand_item(item), use it.
if provider and hasattr(provider, "expand_item") and callable(provider.expand_item): if provider and hasattr(provider, "expand_item") and callable(provider.expand_item):
@@ -566,7 +568,7 @@ class Download_File(Cmdlet):
) -> tuple[int, int]: ) -> tuple[int, int]:
downloaded_count = 0 downloaded_count = 0
queued_magnet_submissions = 0 queued_magnet_submissions = 0
get_search_plugin = registry.get("get_search_plugin") get_provider = registry.get("get_plugin")
SearchResult = registry.get("SearchResult") SearchResult = registry.get("SearchResult")
expanded_items = self._expand_provider_items( expanded_items = self._expand_provider_items(
@@ -622,15 +624,15 @@ class Download_File(Cmdlet):
transfer_label = label transfer_label = label
# If this looks like a provider item and providers are available, prefer provider.download() # If this looks like a plugin-owned item and a plugin is available, prefer plugin.download().
downloaded_path: Optional[Path] = None downloaded_path: Optional[Path] = None
attempted_provider_download = False attempted_provider_download = False
provider_sr = None provider_sr = None
provider_obj = None provider_obj = None
provider_key = self._provider_key_from_item(item) provider_key = self._provider_key_from_item(item)
if provider_key and get_search_plugin and SearchResult: if provider_key and get_provider and SearchResult:
# Reuse helper to derive the provider key from table/provider/source hints. # Reuse helper to derive the plugin key from table/plugin/source hints.
provider_obj = get_search_plugin(provider_key, config) provider_obj = get_provider(provider_key, config)
if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False): if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False):
try: try:
@@ -697,7 +699,7 @@ class Download_File(Cmdlet):
) )
continue continue
# Allow providers to add/enrich tags and metadata during download. # Allow plugins to add or enrich tags and metadata during download.
if provider_sr is not None: if provider_sr is not None:
try: try:
sr_md = getattr(provider_sr, "full_metadata", None) sr_md = getattr(provider_sr, "full_metadata", None)
@@ -838,9 +840,9 @@ class Download_File(Cmdlet):
notes: Optional[Dict[str, str]] = None notes: Optional[Dict[str, str]] = None
try: try:
if isinstance(full_metadata, dict): if isinstance(full_metadata, dict):
# Providers attach pre-built notes under the generic "_notes" key # Plugins attach pre-built notes under the generic "_notes" key
# (e.g. Tidal sets {"lyric": subtitles} during download enrichment). # (e.g. Tidal sets {"lyric": subtitles} during download enrichment).
# This keeps provider-specific metadata handling inside the provider. # This keeps plugin-specific metadata handling inside the plugin.
_provider_notes = full_metadata.get("_notes") _provider_notes = full_metadata.get("_notes")
if isinstance(_provider_notes, dict) and _provider_notes: if isinstance(_provider_notes, dict) and _provider_notes:
notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v} notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v}
@@ -949,7 +951,6 @@ class Download_File(Cmdlet):
return { return {
"get_plugin": getattr(provider_registry, "get_plugin", None), "get_plugin": getattr(provider_registry, "get_plugin", None),
"get_search_plugin": getattr(provider_registry, "get_search_plugin", None),
"match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None), "match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
"list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None), "list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
"SearchResult": SearchResult, "SearchResult": SearchResult,
@@ -957,7 +958,6 @@ class Download_File(Cmdlet):
except Exception: except Exception:
return { return {
"get_plugin": None, "get_plugin": None,
"get_search_plugin": None,
"match_plugin_name_for_url": None, "match_plugin_name_for_url": None,
"list_selection_url_prefixes": None, "list_selection_url_prefixes": None,
"SearchResult": None, "SearchResult": None,
+22 -22
View File
@@ -15,10 +15,10 @@ import sys
from SYS.logger import log, debug from SYS.logger import log, debug
from plugins.metadata_provider import ( from plugins.metadata_provider import (
get_default_subject_scrape_provider, get_default_subject_scrape_plugin,
get_metadata_provider, get_metadata_plugin,
get_metadata_provider_for_url, get_metadata_plugin_for_url,
list_metadata_providers, list_metadata_plugins,
scrape_isbn_metadata, scrape_isbn_metadata,
scrape_openlibrary_metadata, scrape_openlibrary_metadata,
) )
@@ -393,33 +393,33 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
scrape_url = parsed_args.get("scrape") scrape_url = parsed_args.get("scrape")
scrape_requested = scrape_flag_present or scrape_url is not None scrape_requested = scrape_flag_present or scrape_url is not None
# Handle URL or provider scraping mode. # Handle URL or metadata-plugin scraping mode.
if scrape_requested: if scrape_requested:
import json as json_module import json as json_module
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else "" scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
provider = None plugin = None
if scrape_target.startswith(("http://", "https://")): if scrape_target.startswith(("http://", "https://")):
provider = get_metadata_provider_for_url(scrape_target, config) plugin = get_metadata_plugin_for_url(scrape_target, config)
if provider is None: if plugin is None:
log("No metadata provider can scrape this URL", file=sys.stderr) log("No metadata plugin can scrape this URL", file=sys.stderr)
return 1 return 1
payload = provider.scrape_url_payload(scrape_target) payload = plugin.scrape_url_payload(scrape_target)
if not isinstance(payload, dict): if not isinstance(payload, dict):
log(f"No metadata extracted from URL via {provider.name}", file=sys.stderr) log(f"No metadata extracted from URL via {plugin.name}", file=sys.stderr)
return 1 return 1
print(json_module.dumps(payload, ensure_ascii=False)) print(json_module.dumps(payload, ensure_ascii=False))
return 0 return 0
if scrape_target: if scrape_target:
provider = get_metadata_provider(scrape_target, config) plugin = get_metadata_plugin(scrape_target, config)
else: else:
provider = get_default_subject_scrape_provider(config) plugin = get_default_subject_scrape_plugin(config)
if provider is None: if plugin is None:
if scrape_target: if scrape_target:
log(f"Unknown metadata provider: {scrape_target}", file=sys.stderr) log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
else: else:
log("No default metadata provider is available for subject scraping", file=sys.stderr) log("No default metadata plugin is available for subject scraping", file=sys.stderr)
return 1 return 1
backend = None backend = None
@@ -548,7 +548,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
query_hint = resolved_subject_query or identifier_query or combined_query or title_hint query_hint = resolved_subject_query or identifier_query or combined_query or title_hint
if not query_hint: if not query_hint:
log( log(
f"No query could be resolved for metadata provider '{provider.name}'", f"No query could be resolved for metadata plugin '{provider.name}'",
file=sys.stderr file=sys.stderr
) )
return 1 return 1
@@ -749,9 +749,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
) )
return 0 return 0
provider_for_apply = get_metadata_provider(str(result_provider), config) plugin_for_apply = get_metadata_plugin(str(result_provider), config)
if provider_for_apply is not None: if plugin_for_apply is not None:
apply_tags = provider_for_apply.filter_tags_for_store_apply( apply_tags = plugin_for_apply.filter_tags_for_store_apply(
[str(t) for t in result_tags if t is not None] [str(t) for t in result_tags if t is not None]
) )
else: else:
@@ -946,7 +946,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
_SCRAPE_CHOICES = [] _SCRAPE_CHOICES = []
try: try:
_SCRAPE_CHOICES = sorted(list_metadata_providers().keys()) _SCRAPE_CHOICES = sorted(list_metadata_plugins().keys())
except Exception: except Exception:
_SCRAPE_CHOICES = [ _SCRAPE_CHOICES = [
"itunes", "itunes",
@@ -1000,7 +1000,7 @@ class Get_Tag(Cmdlet):
' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")', ' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")',
" -store: Store result to key for downstream pipeline", " -store: Store result to key for downstream pipeline",
" -emit: Quiet mode (no interactive selection)", " -emit: Quiet mode (no interactive selection)",
" -scrape: Scrape metadata from URL or metadata provider", " -scrape: Scrape metadata from URL or metadata plugin",
], ],
exec=self.run, exec=self.run,
) )
+37 -17
View File
@@ -1,4 +1,4 @@
"""search-file cmdlet: Search for files in storage backends (Hydrus).""" """search-file cmdlet: Search store backends and search-capable plugins."""
from __future__ import annotations from __future__ import annotations
@@ -15,11 +15,11 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import log, debug, debug_panel from SYS.logger import log, debug, debug_panel
from SYS.payload_builders import build_file_result_payload, normalize_file_extension from SYS.payload_builders import build_file_result_payload, normalize_file_extension
from ProviderCore.registry import get_search_plugin, list_search_plugins from ProviderCore.registry import get_plugin_with_capability, list_plugins_with_capability
from SYS.rich_display import ( from SYS.rich_display import (
show_provider_config_panel, show_plugin_config_panel,
show_store_config_panel, show_store_config_panel,
show_available_providers_panel, show_available_plugins_panel,
) )
from SYS.database import insert_worker, update_worker, append_worker_stdout from SYS.database import insert_worker, update_worker, append_worker_stdout
from SYS.item_accessors import get_extension_field, get_int_field, get_result_title from SYS.item_accessors import get_extension_field, get_int_field, get_result_title
@@ -164,13 +164,13 @@ def _summarize_worker_results(results: Sequence[Dict[str, Any]], preview_limit:
class search_file(Cmdlet): class search_file(Cmdlet):
"""Class-based search-file cmdlet for searching storage backends.""" """Class-based search-file cmdlet for searching backends and providers."""
def __init__(self) -> None: def __init__(self) -> None:
super().__init__( super().__init__(
name="search-file", name="search-file",
summary="Search storage backends (Hydrus) or external plugins (via -plugin).", summary="Search configured store backends or search-capable plugins.",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-plugin NAME]", usage="search-file [-query <query>] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]",
arg=[ arg=[
CmdletArg( CmdletArg(
"limit", "limit",
@@ -178,6 +178,7 @@ class search_file(Cmdlet):
description="Limit results (default: 100)" description="Limit results (default: 100)"
), ),
SharedArgs.STORE, SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.QUERY, SharedArgs.QUERY,
SharedArgs.PLUGIN, SharedArgs.PLUGIN,
CmdletArg( CmdletArg(
@@ -187,8 +188,10 @@ class search_file(Cmdlet):
), ),
], ],
detail=[ detail=[
"Search across storage backends: Hydrus instances", "Search across configured store backends or plugin providers.",
"Use -store to search a specific backend by name", "Use -store to target a specific store backend by name.",
"Use -plugin with -instance to target a named provider config.",
"In plugin mode, -store <name> is kept as a compatibility alias for -instance <name>.",
"URL search: url:* (any URL) or url:<value> (URL substring)", "URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)", "Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png", "Hydrus-style extension: system:filetype = png",
@@ -207,6 +210,7 @@ class search_file(Cmdlet):
"", "",
"Plugin search (-plugin):", "Plugin search (-plugin):",
"search-file -plugin youtube 'tutorial' # Search YouTube plugin", "search-file -plugin youtube 'tutorial' # Search YouTube plugin",
"search-file -plugin ftp -instance work '*' # Search a named FTP/SCP plugin instance",
"search-file -plugin alldebrid '*' # List AllDebrid magnets", "search-file -plugin alldebrid '*' # List AllDebrid magnets",
"search-file -plugin alldebrid -open 123 '*' # Show files for a magnet", "search-file -plugin alldebrid -open 123 '*' # Show files for a magnet",
], ],
@@ -1451,6 +1455,7 @@ class search_file(Cmdlet):
self, self,
*, *,
plugin_name: str, plugin_name: str,
instance_name: Optional[str],
query: str, query: str,
limit: int, limit: int,
limit_set: bool, limit_set: bool,
@@ -1475,15 +1480,15 @@ class search_file(Cmdlet):
log("Error: search-file -plugin requires both plugin and query", file=sys.stderr) log("Error: search-file -plugin requires both plugin and query", file=sys.stderr)
log(f"Usage: {self.usage}", file=sys.stderr) log(f"Usage: {self.usage}", file=sys.stderr)
providers_map = list_search_plugins(config) providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a] available = [n for n, a in providers_map.items() if a]
unconfigured = [n for n, a in providers_map.items() if not a] unconfigured = [n for n, a in providers_map.items() if not a]
if unconfigured: if unconfigured:
show_provider_config_panel(unconfigured) show_plugin_config_panel(unconfigured)
if available: if available:
show_available_providers_panel(available) show_available_plugins_panel(available)
return 1 return 1
@@ -1496,7 +1501,7 @@ class search_file(Cmdlet):
if hasattr(ctx_mod, "get_pipeline_state"): if hasattr(ctx_mod, "get_pipeline_state"):
progress = ctx_mod.get_pipeline_state().live_progress progress = ctx_mod.get_pipeline_state().live_progress
provider = get_search_plugin(plugin_name, config) provider = get_plugin_with_capability(plugin_name, "search", config)
if not provider: if not provider:
if progress: if progress:
try: try:
@@ -1504,12 +1509,12 @@ class search_file(Cmdlet):
except Exception: except Exception:
pass pass
show_provider_config_panel([plugin_name]) show_plugin_config_panel([plugin_name])
providers_map = list_search_plugins(config) providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a] available = [n for n, a in providers_map.items() if a]
if available: if available:
show_available_providers_panel(available) show_available_plugins_panel(available)
return 1 return 1
worker_id = str(uuid.uuid4()) worker_id = str(uuid.uuid4())
@@ -1542,6 +1547,8 @@ class search_file(Cmdlet):
normalized_query = (normalized_query or "").strip() normalized_query = (normalized_query or "").strip()
query = normalized_query or "*" query = normalized_query or "*"
search_filters = dict(provider_filters or {}) search_filters = dict(provider_filters or {})
if instance_name and not search_filters.get("instance"):
search_filters["instance"] = str(instance_name).strip()
# Dynamic table generation via provider # Dynamic table generation via provider
table_title = provider.get_table_title(query, search_filters).strip().rstrip(":") table_title = provider.get_table_title(query, search_filters).strip().rstrip(":")
@@ -1564,6 +1571,7 @@ class search_file(Cmdlet):
"search-file provider request", "search-file provider request",
[ [
("provider", plugin_name), ("provider", plugin_name),
("instance", search_filters.get("instance") or "<default>"),
("query", query), ("query", query),
("limit", limit), ("limit", limit),
("filters", search_filters or "<none>"), ("filters", search_filters or "<none>"),
@@ -1581,7 +1589,7 @@ class search_file(Cmdlet):
border_style="cyan", border_style="cyan",
) )
# Allow providers to apply provider-specific UX transforms (e.g. auto-expansion) # Allow plugins to apply plugin-specific UX transforms (e.g. auto-expansion)
try: try:
post = getattr(provider, "postprocess_search_results", None) post = getattr(provider, "postprocess_search_results", None)
if callable(post) and isinstance(results, list): if callable(post) and isinstance(results, list):
@@ -1737,6 +1745,10 @@ class search_file(Cmdlet):
f.lower() f.lower()
for f in (flag_registry.get("store") or {"-store", "--store"}) for f in (flag_registry.get("store") or {"-store", "--store"})
} }
instance_flags = {
f.lower()
for f in (flag_registry.get("instance") or {"-instance", "--instance"})
}
limit_flags = { limit_flags = {
f.lower() f.lower()
for f in (flag_registry.get("limit") or {"-limit", "--limit"}) for f in (flag_registry.get("limit") or {"-limit", "--limit"})
@@ -1753,6 +1765,7 @@ class search_file(Cmdlet):
# Parse arguments # Parse arguments
query = "" query = ""
storage_backend: Optional[str] = None storage_backend: Optional[str] = None
instance_name: Optional[str] = None
plugin_name: Optional[str] = None plugin_name: Optional[str] = None
open_id: Optional[int] = None open_id: Optional[int] = None
limit = 100 limit = 100
@@ -1773,6 +1786,10 @@ class search_file(Cmdlet):
plugin_name = args_list[i + 1] plugin_name = args_list[i + 1]
i += 2 i += 2
continue continue
if low in instance_flags and i + 1 < len(args_list):
instance_name = args_list[i + 1]
i += 2
continue
if low in open_flags and i + 1 < len(args_list): if low in open_flags and i + 1 < len(args_list):
try: try:
open_id = int(args_list[i + 1]) open_id = int(args_list[i + 1])
@@ -1804,8 +1821,11 @@ class search_file(Cmdlet):
query = query.strip() query = query.strip()
if plugin_name: if plugin_name:
if storage_backend and not instance_name:
instance_name = storage_backend
return self._run_plugin_search( return self._run_plugin_search(
plugin_name=plugin_name, plugin_name=plugin_name,
instance_name=instance_name,
query=query, query=query,
limit=limit, limit=limit,
limit_set=limit_set, limit_set=limit_set,
+1 -1
View File
@@ -493,7 +493,7 @@ def _maybe_download_hydrus_file(item: Any,
""" """
try: try:
from SYS.config import get_hydrus_access_key, get_hydrus_url from SYS.config import get_hydrus_access_key, get_hydrus_url
from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient, download_hydrus_file
# Prefer per-item Hydrus instance name when it matches a configured instance. # Prefer per-item Hydrus instance name when it matches a configured instance.
store_name = None store_name = None
+170 -98
View File
@@ -10,7 +10,7 @@ from datetime import datetime, timedelta
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs
from pathlib import Path from pathlib import Path
from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args
from ProviderCore.registry import get_plugin_for_url from ProviderCore.registry import get_plugin, get_plugin_for_url
from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream
from SYS.result_table import Table from SYS.result_table import Table
from MPV.mpv_ipc import MPV from MPV.mpv_ipc import MPV
@@ -32,6 +32,13 @@ _IPV4_RE = re.compile(r"^\d+\.\d+\.\d+\.\d+$")
_MPD_PATH_RE = re.compile(r"\.mpd($|\?)") _MPD_PATH_RE = re.compile(r"\.mpd($|\?)")
def _get_hydrus_provider(config: Optional[Dict[str, Any]] = None) -> Any:
try:
return get_plugin("hydrusnetwork", config or {})
except Exception:
return None
def _repo_root() -> Path: def _repo_root() -> Path:
try: try:
return Path(__file__).resolve().parent.parent return Path(__file__).resolve().parent.parent
@@ -575,20 +582,51 @@ def _send_ipc_command(command: Dict[str, Any], silent: bool = False, wait: bool
return None return None
def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]: def _extract_store_and_hash(
item: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> tuple[Optional[str], Optional[str]]:
store: Optional[str] = None store: Optional[str] = None
file_hash: Optional[str] = None file_hash: Optional[str] = None
targets: list[str] = []
def _add_target(value: Any) -> None:
text = str(value or "").strip()
if text:
targets.append(text)
try: try:
if isinstance(item, dict): if isinstance(item, dict):
store = item.get("store") store = item.get("store")
file_hash = item.get("hash") or item.get("file_hash") file_hash = item.get("hash") or item.get("file_hash")
_add_target(item.get("path"))
_add_target(item.get("url"))
_add_target(item.get("filename"))
metadata = item.get("full_metadata") or item.get("metadata")
else: else:
store = getattr(item, "store", None) store = getattr(item, "store", None)
file_hash = getattr(item, "hash", None) or getattr(item, "file_hash", None) file_hash = getattr(item, "hash", None) or getattr(item, "file_hash", None)
_add_target(getattr(item, "path", None))
_add_target(getattr(item, "url", None))
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
except Exception: except Exception:
store = None store = None
file_hash = None file_hash = None
metadata = None
if isinstance(metadata, dict):
try:
if not store:
store = metadata.get("store")
if not file_hash or str(file_hash).strip().lower() in {"unknown", "none", "n/a", "na"}:
file_hash = metadata.get("hash") or metadata.get("hash_hex")
_add_target(metadata.get("path"))
_add_target(metadata.get("url"))
_add_target(metadata.get("selection_url"))
_add_target(metadata.get("hydrus_url"))
except Exception:
pass
try: try:
store = str(store).strip() if store else None store = str(store).strip() if store else None
@@ -600,17 +638,37 @@ def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]:
except Exception: except Exception:
file_hash = None file_hash = None
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is not None:
normalized_store = None
try:
if store and hydrus_provider.is_store_name(store):
normalized_store = store
except Exception:
normalized_store = None
for target in targets:
try:
parsed_store, parsed_hash = hydrus_provider.parse_hydrus_url(target)
except Exception:
parsed_store, parsed_hash = None, ""
if parsed_hash and not file_hash:
file_hash = parsed_hash
if parsed_store:
normalized_store = parsed_store
if normalized_store:
store = normalized_store
elif store and store.upper() in {"PATH", "LOCAL", "UNKNOWN"}:
store = None
if not file_hash: if not file_hash:
try: try:
text = None for text in targets:
if isinstance(item, dict):
text = item.get("path") or item.get("url") or item.get("filename")
else:
text = getattr(item, "path", None) or getattr(item, "url", None)
if text:
m = _SHA256_RE.search(str(text).lower()) m = _SHA256_RE.search(str(text).lower())
if m: if m:
file_hash = m.group(0) file_hash = m.group(0)
break
except Exception: except Exception:
pass pass
@@ -681,6 +739,8 @@ def _prefetch_notes_async(
set_notes_prefetch_pending(store, file_hash, True) set_notes_prefetch_pending(store, file_hash, True)
registry = Store(cfg, suppress_debug=True) registry = Store(cfg, suppress_debug=True)
if not registry.is_available(str(store)):
return
backend = registry[str(store)] backend = registry[str(store)]
notes = backend.get_note(str(file_hash), config=cfg) or {} notes = backend.get_note(str(file_hash), config=cfg) or {}
store_cached_notes(store, file_hash, notes) store_cached_notes(store, file_hash, notes)
@@ -718,7 +778,7 @@ def _schedule_notes_prefetch(items: Sequence[Any], config: Optional[Dict[str, An
seen: set[str] = set() seen: set[str] = set()
scheduled = 0 scheduled = 0
for item in items or []: for item in items or []:
store, file_hash = _extract_store_and_hash(item) store, file_hash = _extract_store_and_hash(item, config=config)
if not store or not file_hash: if not store or not file_hash:
continue continue
key = f"{store.lower()}:{file_hash}" key = f"{store.lower()}:{file_hash}"
@@ -789,7 +849,12 @@ def _extract_target_from_memory_uri(text: str) -> Optional[str]:
return None return None
def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional[str]: def _find_hydrus_instance_for_hash(
hash_str: str,
file_storage: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Find which Hydrus instance serves a specific file hash. """Find which Hydrus instance serves a specific file hash.
Args: Args:
@@ -799,27 +864,29 @@ def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional
Returns: Returns:
Instance name (e.g., 'home') or None if not found Instance name (e.g., 'home') or None if not found
""" """
# Query each Hydrus backend to see if it has this file hydrus_provider = _get_hydrus_provider(config)
for backend_name in file_storage.list_backends(): if hydrus_provider is None:
backend = file_storage[backend_name] return None
# Check if this is a Hydrus backend by checking class name
backend_class = type(backend).__name__
if backend_class != "HydrusNetwork":
continue
try: try:
# Query metadata to see if this instance has the file for backend_name, _backend in hydrus_provider.iter_backends():
metadata = backend.get_metadata(hash_str) try:
if metadata: if hydrus_provider.hash_exists(hash_str, store_name=str(backend_name)):
return backend_name return str(backend_name)
except Exception: except Exception:
# This instance doesn't have the file or had an error
continue continue
except Exception:
return None
return None return None
def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]: def _find_hydrus_instance_by_url(
url: str,
file_storage: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Find which Hydrus instance matches a given URL. """Find which Hydrus instance matches a given URL.
Args: Args:
@@ -829,30 +896,12 @@ def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]:
Returns: Returns:
Instance name (e.g., 'home') or None if not found Instance name (e.g., 'home') or None if not found
""" """
from urllib.parse import urlparse hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is None:
parsed_target = urlparse(url) return None
target_netloc = parsed_target.netloc.lower()
# Check each Hydrus backend's URL
for backend_name in file_storage.list_backends():
backend = file_storage[backend_name]
backend_class = type(backend).__name__
if backend_class != "HydrusNetwork":
continue
# Get the backend's base URL from its client
try: try:
backend_url = backend._client.base_url return hydrus_provider.match_store_name_for_url(url)
parsed_backend = urlparse(backend_url)
backend_netloc = parsed_backend.netloc.lower()
# Match by netloc (host:port)
if target_netloc == backend_netloc:
return backend_name
except Exception: except Exception:
continue
return None return None
@@ -891,7 +940,8 @@ def _normalize_playlist_path(text: Optional[str]) -> Optional[str]:
def _infer_store_from_playlist_item( def _infer_store_from_playlist_item(
item: Dict[str, item: Dict[str,
Any], Any],
file_storage: Optional[Any] = None file_storage: Optional[Any] = None,
config: Optional[Dict[str, Any]] = None,
) -> str: ) -> str:
"""Infer a friendly store label from an MPV playlist entry. """Infer a friendly store label from an MPV playlist entry.
@@ -915,7 +965,7 @@ def _infer_store_from_playlist_item(
# If we have file_storage, query each Hydrus instance to find which one has this hash # If we have file_storage, query each Hydrus instance to find which one has this hash
if file_storage: if file_storage:
hash_str = target.lower() hash_str = target.lower()
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
return "hydrus" return "hydrus"
@@ -929,7 +979,7 @@ def _infer_store_from_playlist_item(
hash_match = _SHA256_RE.search(target.lower()) hash_match = _SHA256_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(0) hash_str = hash_match.group(0)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
return "hydrus" return "hydrus"
@@ -968,11 +1018,11 @@ def _infer_store_from_playlist_item(
hash_match = _HASH_QUERY_RE.search(target.lower()) hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(1) hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
# If no hash in URL, try matching the base URL to configured instances # If no hash in URL, try matching the base URL to configured instances
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage) hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
return "hydrus" return "hydrus"
@@ -982,10 +1032,10 @@ def _infer_store_from_playlist_item(
hash_match = _HASH_QUERY_RE.search(target.lower()) hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(1) hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage) hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config)
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
return "hydrus" return "hydrus"
@@ -1320,6 +1370,16 @@ def _get_playable_path(
elif isinstance(item, str): elif isinstance(item, str):
path = item path = item
if not store or not file_hash or str(file_hash).strip().lower() == "unknown":
try:
extracted_store, extracted_hash = _extract_store_and_hash(item, config=config)
except Exception:
extracted_store, extracted_hash = None, None
if extracted_store and not store:
store = extracted_store
if extracted_hash and (not file_hash or str(file_hash).strip().lower() == "unknown"):
file_hash = extracted_hash
# Debug: show incoming values # Debug: show incoming values
try: try:
debug(f"_get_playable_path: store={store}, path={path}, hash={file_hash}") debug(f"_get_playable_path: store={store}, path={path}, hash={file_hash}")
@@ -1384,6 +1444,7 @@ def _get_playable_path(
# - MPV IPC pipe (transport) # - MPV IPC pipe (transport)
# - PipeObject (pipeline data) # - PipeObject (pipeline data)
backend_target_resolved = False backend_target_resolved = False
hydrus_provider = _get_hydrus_provider(config)
if store and file_hash and file_hash != "unknown" and file_storage: if store and file_hash and file_hash != "unknown" and file_storage:
try: try:
backend = file_storage[store] backend = file_storage[store]
@@ -1391,18 +1452,14 @@ def _get_playable_path(
backend = None backend = None
if backend is not None: if backend is not None:
backend_class = type(backend).__name__
backend_target_resolved = True backend_target_resolved = True
# HydrusNetwork: build a playable API file URL without browser side-effects. # Hydrus playback should resolve via the provider so store aliases and URL building stay centralized.
if backend_class == "HydrusNetwork": if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(store)):
try: try:
client = getattr(backend, "_client", None) resolved_path = hydrus_provider.build_file_url(file_hash, store_name=str(store))
base_url = getattr(client, "url", None) if resolved_path:
if base_url: path = resolved_path
base_url = str(base_url).rstrip("/")
# Auth is provided via http-header-fields (set in _queue_items).
path = f"{base_url}/get_files/file?hash={file_hash}"
except Exception as e: except Exception as e:
debug( debug(
f"Error building Hydrus URL from store '{store}': {e}", f"Error building Hydrus URL from store '{store}': {e}",
@@ -1570,20 +1627,31 @@ def _queue_items(
item_store = None item_store = None
if isinstance(item, dict): if isinstance(item, dict):
item_store = item.get("store") item_store = item.get("store")
metadata = item.get("full_metadata") or item.get("metadata")
else: else:
item_store = getattr(item, "store", None) item_store = getattr(item, "store", None)
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not item_store and isinstance(metadata, dict):
item_store = metadata.get("store")
if not item_store:
item_store, _ = _extract_store_and_hash(item, config=config)
if item_store: if item_store:
item_store_name = str(item_store).strip() or None item_store_name = str(item_store).strip() or None
if item_store and file_storage: if item_store_name and file_storage:
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is not None:
try: try:
backend = file_storage[str(item_store)] client = hydrus_provider.get_client(
store_name=item_store_name,
allow_default=False,
)
except Exception: except Exception:
backend = None client = None
if client is not None:
if backend is not None and type(backend).__name__ == "HydrusNetwork":
client = getattr(backend, "_client", None)
base_url = getattr(client, "url", None) base_url = getattr(client, "url", None)
key = getattr(client, "access_key", None) key = getattr(client, "access_key", None)
if base_url: if base_url:
@@ -1608,6 +1676,10 @@ def _queue_items(
f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}" f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}"
) )
hydrus_target = bool(
effective_hydrus_header and _is_hydrus_path(str(target), effective_hydrus_url)
)
norm_key = _normalize_playlist_path(target) or str(target).strip().lower() norm_key = _normalize_playlist_path(target) or str(target).strip().lower()
if norm_key in existing_targets or norm_key in new_targets: if norm_key in existing_targets or norm_key in new_targets:
debug(f"Skipping duplicate playlist entry: {title or target}") debug(f"Skipping duplicate playlist entry: {title or target}")
@@ -1616,10 +1688,11 @@ def _queue_items(
# Use memory:// M3U hack to pass title to MPV. # Use memory:// M3U hack to pass title to MPV.
# Avoid this for probable ytdl URLs because it can prevent the hook from triggering. # Avoid this for probable ytdl URLs because it can prevent the hook from triggering.
if title and not _is_probable_ytdl_url(target): safe_title = title.replace("\n", " ").replace("\r", "") if title else None
if title and hydrus_target:
target_to_send = target
elif title and not _is_probable_ytdl_url(target):
# Sanitize title for M3U (remove newlines) # Sanitize title for M3U (remove newlines)
safe_title = title.replace("\n", " ").replace("\r", "")
# Carry the store name for hash URLs so MPV.lyric can resolve the backend. # Carry the store name for hash URLs so MPV.lyric can resolve the backend.
# This is especially important for local file-server URLs like /get_files/file?hash=... # This is especially important for local file-server URLs like /get_files/file?hash=...
target_for_m3u = target target_for_m3u = target
@@ -1646,15 +1719,14 @@ def _queue_items(
# so MPV.lyric can resolve the correct backend for notes. # so MPV.lyric can resolve the correct backend for notes.
if mode == "replace": if mode == "replace":
try: try:
s, h = _extract_store_and_hash(item) s, h = _extract_store_and_hash(item, config=config)
_set_mpv_item_context(s, h) _set_mpv_item_context(s, h)
except Exception: except Exception:
pass pass
# If this is a Hydrus path, set header property and yt-dlp headers before loading. # If this is a Hydrus path, set header property and yt-dlp headers before loading.
# Use the real target (not the memory:// wrapper) for detection. # Use the real target (not the memory:// wrapper) for detection.
if effective_hydrus_header and _is_hydrus_path(str(target), if hydrus_target:
effective_hydrus_url):
header_cmd = { header_cmd = {
"command": "command":
["set_property", ["set_property",
@@ -1683,10 +1755,18 @@ def _queue_items(
except Exception: except Exception:
pass pass
load_options: Dict[str, str] = {}
if hydrus_target and command_name == "loadfile":
load_options["ytdl"] = "no"
if safe_title:
load_options["force-media-title"] = safe_title
command_args: List[Any] = [command_name, target_to_send, mode]
if load_options:
command_args.extend([-1, load_options])
cmd = { cmd = {
"command": [command_name, "command": command_args,
target_to_send,
mode],
"request_id": 200 "request_id": 200
} }
try: try:
@@ -2159,7 +2239,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Prefer the store/hash from the piped item when auto-playing. # Prefer the store/hash from the piped item when auto-playing.
try: try:
s, h = _extract_store_and_hash(items_to_add[0]) s, h = _extract_store_and_hash(items_to_add[0], config=config)
_set_mpv_item_context(s, h) _set_mpv_item_context(s, h)
except Exception: except Exception:
pass pass
@@ -2293,7 +2373,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
else: else:
# Play item # Play item
try: try:
s, h = _extract_store_and_hash(item) s, h = _extract_store_and_hash(item, config=config)
_set_mpv_item_context(s, h) _set_mpv_item_context(s, h)
except Exception: except Exception:
pass pass
@@ -2376,26 +2456,17 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Extract the real path/URL from memory:// wrapper if present # Extract the real path/URL from memory:// wrapper if present
real_path = _extract_target_from_memory_uri(filename) or filename real_path = _extract_target_from_memory_uri(filename) or filename
# Try to extract hash from the path/URL # Try to extract hash/store from the path/URL via provider-aware normalization.
file_hash = None store_name, file_hash = _extract_store_and_hash(
store_name = None {
"path": real_path,
# Check if it's a Hydrus URL "title": title,
if "get_files/file" in real_path or "hash=" in real_path: },
# Extract hash from Hydrus URL config=config,
hash_match = _HASH_QUERY_RE.search(real_path.lower())
if hash_match:
file_hash = hash_match.group(1)
# Try to find which Hydrus instance has this file
if file_storage:
store_name = _find_hydrus_instance_for_hash(
file_hash,
file_storage
) )
if not store_name:
store_name = "hydrus"
# Check if it's a hash-based local file # Check if it's a hash-based local file
elif real_path: if not file_hash and real_path:
# Try to extract hash from filename (e.g., C:\path\1e8c46...a1b2.mp4) # Try to extract hash from filename (e.g., C:\path\1e8c46...a1b2.mp4)
path_obj = Path(real_path) path_obj = Path(real_path)
stem = path_obj.stem # filename without extension stem = path_obj.stem # filename without extension
@@ -2407,7 +2478,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if not store_name: if not store_name:
store_name = _infer_store_from_playlist_item( store_name = _infer_store_from_playlist_item(
item, item,
file_storage=file_storage file_storage=file_storage,
config=config,
) )
# Build PipeObject with proper metadata # Build PipeObject with proper metadata
@@ -2651,7 +2723,7 @@ def _start_mpv(
# target change (the helper may start before playback begins). # target change (the helper may start before playback begins).
try: try:
if items: if items:
s, h = _extract_store_and_hash(items[0]) s, h = _extract_store_and_hash(items[0], config=config)
_set_mpv_item_context(s, h) _set_mpv_item_context(s, h)
except Exception: except Exception:
pass pass
+33 -23
View File
@@ -2,11 +2,11 @@
This walkthrough adds a real bundled `ftp` plugin so users can: This walkthrough adds a real bundled `ftp` plugin so users can:
- run `search-file -plugin ftp ...` - run `search-file -plugin ftp -instance <name> ...`
- browse remote folders as result tables - browse remote folders as result tables
- select file rows to `download-file` - select file rows to `download-file`
- pipe selected file rows into `add-file` - pipe selected file rows into `add-file`
- upload local files with `add-file -plugin ftp` - upload local files with `add-file -plugin ftp -instance <name>`
The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
@@ -20,14 +20,14 @@ The FTP plugin demonstrates the main provider hooks that matter for a storage-st
- `selector()` turns folder rows into a follow-up table when the user runs `@N`. - `selector()` turns folder rows into a follow-up table when the user runs `@N`.
- `download()` and `download_url()` fetch FTP files into `download-file` output paths. - `download()` and `download_url()` fetch FTP files into `download-file` output paths.
- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first. - `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first.
- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server. - `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server.
## Example Config ## Example Config
Add an FTP provider block to your config: Add one or more named FTP provider instances to your config:
```toml ```toml
[provider.ftp] [provider.ftp.work]
host = "ftp.example.com" host = "ftp.example.com"
port = 21 port = 21
username = "demo" username = "demo"
@@ -37,11 +37,20 @@ tls = false
passive = true passive = true
timeout = 20 timeout = 20
search_depth = 1 search_depth = 1
[provider.ftp.archive]
host = "archive.example.com"
port = 2121
username = "archive-bot"
password = "secret"
base_path = "/dropbox"
tls = true
``` ```
Notes: Notes:
- `host` is the only required field for the plugin to validate. - `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`.
- `host` is the only required field for each instance to validate.
- `username` defaults to `anonymous` and `password` defaults to `anonymous@`. - `username` defaults to `anonymous` and `password` defaults to `anonymous@`.
- `base_path` is both the default search root and the upload target directory. - `base_path` is both the default search root and the upload target directory.
- `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default. - `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default.
@@ -51,25 +60,25 @@ Notes:
Basic listing from the configured base path: Basic listing from the configured base path:
```powershell ```powershell
search-file -plugin ftp "*" search-file -plugin ftp -instance work "*"
``` ```
Search by filename fragment: Search by filename fragment:
```powershell ```powershell
search-file -plugin ftp "invoice" search-file -plugin ftp -instance work "invoice"
``` ```
Search a different subtree and recurse deeper: Search a different subtree and recurse deeper:
```powershell ```powershell
search-file -plugin ftp "path:/pub depth:2 invoice" search-file -plugin ftp -instance work "path:/pub depth:2 invoice"
``` ```
Filter to folders only: Filter to folders only:
```powershell ```powershell
search-file -plugin ftp "path:/pub type:folder *" search-file -plugin ftp -instance work "path:/pub type:folder *"
``` ```
The plugin returns rows with explicit columns for name, type, directory, size, and modification time. The plugin returns rows with explicit columns for name, type, directory, size, and modification time.
@@ -79,20 +88,20 @@ The plugin returns rows with explicit columns for name, type, directory, size, a
Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory: Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory:
```powershell ```powershell
search-file -plugin ftp "*" search-file -plugin ftp -instance work "*"
@2 @2
``` ```
File rows carry an explicit row action: File rows carry an explicit row action:
```powershell ```powershell
download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf download-file -plugin ftp -instance work -url ftp://ftp.example.com/incoming/report.pdf
``` ```
That means plain `@N` on a file row downloads it immediately: That means plain `@N` on a file row downloads it immediately:
```powershell ```powershell
search-file -plugin ftp "report" search-file -plugin ftp -instance work "report"
@1 @1
``` ```
@@ -101,14 +110,14 @@ search-file -plugin ftp "report"
If you want the downloaded file in a specific local directory: If you want the downloaded file in a specific local directory:
```powershell ```powershell
search-file -plugin ftp "report" search-file -plugin ftp -instance work "report"
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
``` ```
If you want to ingest the selected FTP file into a configured store backend: If you want to ingest the selected FTP file into a configured store backend:
```powershell ```powershell
search-file -plugin ftp "report" search-file -plugin ftp -instance work "report"
@1 | add-file -store tutorial @1 | add-file -store tutorial
``` ```
@@ -117,23 +126,24 @@ Why this works:
- the file row advertises a `download-file` row action - the file row advertises a `download-file` row action
- the pipeline auto-inserts that download before `add-file` - the pipeline auto-inserts that download before `add-file`
- the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion - the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion
- file rows also carry the chosen `instance`, so selection replay and `@N | add-file ...` keep the same FTP target
## Upload Flow ## Upload Flow
Uploading uses the same provider name, but through `add-file -plugin ftp`: Uploading uses the same provider name, but through `add-file -plugin ftp -instance <name>`:
```powershell ```powershell
add-file -plugin ftp -path C:\Media\report.pdf add-file -plugin ftp -instance archive -path C:\Media\report.pdf
``` ```
That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result. That sends the file to the selected instance's FTP `base_path` and returns the FTP URL as the uploaded result.
## Why The Row Metadata Matters ## Why The Row Metadata Matters
The critical part of this plugin is the file-row metadata: The critical part of this plugin is the file-row metadata:
- file rows emit `_selection_args` as `['-url', '<ftp-url>']` - file rows emit `_selection_args` as `['-instance', '<name>', '-url', '<ftp-url>']`
- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '<ftp-url>']` - file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-instance', '<name>', '-url', '<ftp-url>']`
- folder rows do not emit a download action, so `selector()` can own drill-in behavior instead - folder rows do not emit a download action, so `selector()` can own drill-in behavior instead
That split is what keeps these two user experiences compatible: That split is what keeps these two user experiences compatible:
@@ -155,10 +165,10 @@ The code is intentionally small and uses only Python stdlib pieces:
## Recommended Commands To Demo The Walkthrough ## Recommended Commands To Demo The Walkthrough
```powershell ```powershell
search-file -plugin ftp "*" search-file -plugin ftp -instance work "*"
search-file -plugin ftp "path:/incoming depth:2 *.pdf" search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf"
@1 @1
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
@1 | add-file -store tutorial @1 | add-file -store tutorial
add-file -plugin ftp -path C:\Media\report.pdf add-file -plugin ftp -instance archive -path C:\Media\report.pdf
``` ```
+28 -18
View File
@@ -11,16 +11,16 @@ The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py).
The SCP plugin mirrors the FTP walkthrough, but on top of SSH: The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
- `search-file -plugin scp ...` lists remote files and folders over SFTP. - `search-file -plugin scp -instance <name> ...` lists remote files and folders over SFTP.
- plain `@N` on a folder drills into that directory. - plain `@N` on a folder drills into that directory.
- plain `@N` on a file runs `download-file -plugin scp -url ...`. - plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`.
- `@N | add-file -store ...` downloads first, then ingests the local temp file. - `@N | add-file -store ...` downloads first, then ingests the local temp file.
- `add-file -plugin scp -path ...` uploads a local file to the configured remote path. - `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path.
## Example Config ## Example Config
```toml ```toml
[provider.scp] [provider.scp.work]
host = "ssh.example.com" host = "ssh.example.com"
port = 22 port = 22
username = "deploy" username = "deploy"
@@ -31,11 +31,20 @@ timeout = 20
search_depth = 1 search_depth = 1
allow_agent = true allow_agent = true
look_for_keys = true look_for_keys = true
[provider.scp.archive]
host = "ssh-archive.example.com"
port = 2222
username = "archive"
key_path = "C:/Users/Admin/.ssh/archive_ed25519"
base_path = "/srv/archive"
timeout = 20
``` ```
Notes: Notes:
- `host` and `username` are required for the plugin to validate. - `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`.
- `host` and `username` are required for each instance to validate.
- You can use password auth, key auth, or both. - You can use password auth, key auth, or both.
- `base_path` is both the default search root and the default upload directory. - `base_path` is both the default search root and the default upload directory.
@@ -44,25 +53,25 @@ Notes:
List the configured base path: List the configured base path:
```powershell ```powershell
search-file -plugin scp "*" search-file -plugin scp -instance work "*"
``` ```
Search by filename: Search by filename:
```powershell ```powershell
search-file -plugin scp "invoice" search-file -plugin scp -instance work "invoice"
``` ```
Search another subtree with deeper recursion: Search another subtree with deeper recursion:
```powershell ```powershell
search-file -plugin scp "path:/srv/files/releases depth:2 *.zip" search-file -plugin scp -instance work "path:/srv/files/releases depth:2 *.zip"
``` ```
Show only folders: Show only folders:
```powershell ```powershell
search-file -plugin scp "path:/srv/files type:folder *" search-file -plugin scp -instance work "path:/srv/files type:folder *"
``` ```
## Selection Flow ## Selection Flow
@@ -70,21 +79,21 @@ search-file -plugin scp "path:/srv/files type:folder *"
Folder rows are navigation rows: Folder rows are navigation rows:
```powershell ```powershell
search-file -plugin scp "*" search-file -plugin scp -instance work "*"
@2 @2
``` ```
File rows carry an explicit row action, so terminal selection downloads directly: File rows carry an explicit row action, so terminal selection downloads directly:
```powershell ```powershell
search-file -plugin scp "report" search-file -plugin scp -instance work "report"
@1 @1
``` ```
That expands to the equivalent of: That expands to the equivalent of:
```powershell ```powershell
download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf download-file -plugin scp -instance work -url scp://ssh.example.com/srv/files/report.pdf
``` ```
## Download And Add-File Flow ## Download And Add-File Flow
@@ -92,14 +101,14 @@ download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf
Download into a local folder: Download into a local folder:
```powershell ```powershell
search-file -plugin scp "report" search-file -plugin scp -instance work "report"
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
``` ```
Ingest a selected remote file into a configured store backend: Ingest a selected remote file into a configured store backend:
```powershell ```powershell
search-file -plugin scp "report" search-file -plugin scp -instance work "report"
@1 | add-file -store tutorial @1 | add-file -store tutorial
``` ```
@@ -108,13 +117,14 @@ Why this works:
- file rows advertise `_selection_action` for `download-file` - file rows advertise `_selection_action` for `download-file`
- `add-file` selection replay inserts that provider download stage before ingest - `add-file` selection replay inserts that provider download stage before ingest
- the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows - the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows
- file rows also carry the chosen `instance`, so replay stays bound to the same SSH target
## Upload Flow ## Upload Flow
Upload a local file to the configured remote `base_path`: Upload a local file to the configured remote `base_path`:
```powershell ```powershell
add-file -plugin scp -path C:\Media\report.pdf add-file -plugin scp -instance archive -path C:\Media\report.pdf
``` ```
## Implementation Notes ## Implementation Notes
@@ -127,10 +137,10 @@ The plugin uses SFTP for directory listing because SCP itself is a transfer prot
## Recommended Demo Commands ## Recommended Demo Commands
```powershell ```powershell
search-file -plugin scp "*" search-file -plugin scp -instance work "*"
search-file -plugin scp "path:/srv/files depth:2 *.zip" search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip"
@1 @1
@1 | download-file -path C:\Downloads @1 | download-file -path C:\Downloads
@1 | add-file -store tutorial @1 | add-file -store tutorial
add-file -plugin scp -path C:\Media\report.pdf add-file -plugin scp -instance archive -path C:\Media\report.pdf
``` ```
+4 -3
View File
@@ -52,8 +52,9 @@ class MyPlugin(Provider):
Bundled walkthrough: Bundled walkthrough:
- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`; plugin-mode `-store <name>` remains a compatibility alias while store-backed flows still use real backend stores.
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). - The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads. - The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). - The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads. - The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly. - The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives alongside it in [plugins/hydrusnetwork/api.py](plugins/hydrusnetwork/api.py), while [API/HydrusNetwork.py](API/HydrusNetwork.py) remains a compatibility shim. The provider delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
+77 -3
View File
@@ -540,7 +540,7 @@ def download_magnet(
def expand_folder_item( def expand_folder_item(
item: Any, item: Any,
get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]], get_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
config: Dict[str, Any], config: Dict[str, Any],
) -> Tuple[List[Any], Optional[str]]: ) -> Tuple[List[Any], Optional[str]]:
table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table") table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table")
@@ -564,10 +564,10 @@ def expand_folder_item(
except Exception: except Exception:
magnet_id = None magnet_id = None
if magnet_id is None or get_search_plugin is None: if magnet_id is None or get_plugin is None:
return [], None return [], None
plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None plugin = get_plugin("alldebrid", config) if get_plugin else None
if plugin is None: if plugin is None:
return [], None return [], None
@@ -1774,6 +1774,80 @@ class AllDebrid(TableProviderMixin, Provider):
return True return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
_item, payload, meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if not isinstance(payload, dict):
return False
title = str(payload.get("title") or meta.get("name") or "").strip() or "AllDebrid Item"
magnet_name = str(meta.get("magnet_name") or payload.get("detail") or "").strip()
magnet_id = meta.get("magnet_id")
relpath = str(meta.get("relpath") or "").strip()
direct_url = str(payload.get("path") or "").strip()
selection_url = ""
action = meta.get("_selection_action") or meta.get("selection_action")
if isinstance(action, (list, tuple)):
action_tokens = [str(x) for x in action if x is not None]
for idx, token in enumerate(action_tokens):
if str(token).strip().lower() in {"-url", "--url"} and idx + 1 < len(action_tokens):
selection_url = str(action_tokens[idx + 1] or "").strip()
break
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
payload,
title=title,
store=self.name,
path=direct_url or None,
tags=meta.get("tag") or meta.get("tags"),
extra_fields={
"Plugin": self.name,
"Magnet": magnet_name or None,
"Magnet ID": magnet_id,
"Relative Path": relpath or None,
"View": str(meta.get("provider_view") or meta.get("view") or (table_metadata or {}).get("view") or "").strip() or None,
"Direct Url": direct_url or None,
"Selection Url": selection_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=payload,
title=f"AllDebrid Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Magnet", "Magnet ID", "Relative Path", "View", "Path", "File", "Folder", "ID", "Direct URL", "Selection URL", "Plugin"],
value_case="preserve",
)
try: try:
from SYS.result_table_adapters import register_plugin from SYS.result_table_adapters import register_plugin
+256 -4
View File
@@ -1,7 +1,259 @@
"""Plugin-namespace import shim for the strict adapter example module. """Example plugin that uses the new `ResultTable` API.
This keeps example docs pointing at the plugin namespace while the original This module demonstrates a minimal provider adapter that yields `ResultModel`
implementation remains in ``Provider.example_provider`` for compatibility. instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
(`render_table`) for demonstration.
Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default
""" """
from __future__ import annotations
from Provider.example_provider import * # noqa: F401,F403 from pathlib import Path
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column
SAMPLE_ITEMS = [
{
"name": "Book of Awe.pdf",
"path": "sample/Book of Awe.pdf",
"ext": "pdf",
"size": 1024000,
"source": "example",
},
{
"name": "Song of Joy.mp3",
"path": "sample/Song of Joy.mp3",
"ext": "mp3",
"size": 5120000,
"source": "example",
},
{
"name": "Cover Image.jpg",
"path": "sample/Cover Image.jpg",
"ext": "jpg",
"size": 20480,
"source": "example",
},
]
def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]:
"""Convert provider-specific items into `ResultModel` instances.
This adapter enforces the strict API requirement: it yields only
`ResultModel` instances (no legacy dict objects).
"""
for it in items:
title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "")
yield ResultModel(
title=str(title),
path=str(it.get("path")) if it.get("path") else None,
ext=str(it.get("ext")) if it.get("ext") else None,
size_bytes=int(it.get("size")) if it.get("size") is not None else None,
metadata=dict(it),
source=str(it.get("source")) if it.get("source") else "example",
)
# Columns are intentionally *not* mandated. Create a factory that inspects
# sample rows and builds only columns that make sense for the provider data.
from SYS.result_table_api import metadata_column
def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]:
cols: List[ColumnSpec] = [title_column()]
# If any row has an extension, include Ext column
if any(getattr(r, "ext", None) for r in rows):
cols.append(ext_column())
# If any row has size, include Size column
if any(getattr(r, "size_bytes", None) for r in rows):
cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v)))
# Add any top-level metadata keys discovered (up to 3) as optional columns
seen_keys = []
for r in rows:
for k in (r.metadata or {}).keys():
if k in ("name", "title", "path"):
continue
if k not in seen_keys:
seen_keys.append(k)
if len(seen_keys) >= 3:
break
if len(seen_keys) >= 3:
break
for k in seen_keys:
cols.append(metadata_column(k))
return cols
# Selection function: cmdlets rely on this to build selector args when the user
# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer
# -path if available, otherwise fall back to -title.
def selection_fn(row: ResultModel) -> List[str]:
if row.path:
return ["-path", row.path]
return ["-title", row.title]
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
selection_fn=selection_fn,
metadata={"description": "Example provider demonstrating dynamic columns and selectors"},
)
def _format_size(size: Any) -> str:
try:
s = int(size)
except Exception:
return ""
if s >= 1024 ** 3:
return f"{s / (1024 ** 3):.2f} GB"
if s >= 1024 ** 2:
return f"{s / (1024 ** 2):.2f} MB"
if s >= 1024:
return f"{s / 1024:.2f} KB"
return f"{s} B"
def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str:
"""Render a simple ASCII table of `rows` using `columns`.
This is intentionally very small and dependency-free for demonstration.
Renderers in the project should implement the `Renderer` protocol.
"""
rows = list(rows)
# Build cell matrix (strings)
matrix: List[List[str]] = []
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
matrix.append(cells)
# Compute column widths as max(header, content)
headers = [c.header for c in columns]
widths = [len(h) for h in headers]
for row_cells in matrix:
for i, cell in enumerate(row_cells):
widths[i] = max(widths[i], len(cell))
# Helper to format a row
def fmt_row(cells: List[str]) -> str:
return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
lines: List[str] = []
lines.append(fmt_row(headers))
lines.append("-+-".join("-" * w for w in widths))
for row_cells in matrix:
lines.append(fmt_row(row_cells))
return "\n".join(lines)
# Rich-based renderer (returns a Rich Table renderable)
def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]):
"""Render rows as a `rich.table.Table` for terminal output.
Returns the Table object; callers may `Console.print(table)` to render.
"""
try:
from rich.table import Table as RichTable
except Exception as exc: # pragma: no cover - rare if rich missing
raise RuntimeError("rich is required for rich renderer") from exc
table = RichTable(show_header=True, header_style="bold")
for col in columns:
table.add_column(col.header)
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
table.add_row(*cells)
return table
def demo() -> None:
rows = list(adapter(SAMPLE_ITEMS))
table = render_table_rich(rows, columns_factory(rows))
try:
from rich.console import Console
except Exception:
# Fall back to plain printing if rich is not available
print("Example provider output:")
print(render_table(rows, columns_factory(rows)))
return
console = Console()
console.print("Example provider output:")
console.print(table)
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_plugin
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
# Render
try:
from rich.console import Console
except Exception:
print(render_table(rows, cols))
sel_args = provider.selection_args(rows[idx])
print("Selection args for row", idx, "->", sel_args)
return
console = Console()
console.print("Example provider output:")
console.print(render_table_rich(rows, cols))
# Selection args example
sel = provider.selection_args(rows[idx])
console.print("Selection args for row", idx, "->", sel)
if __name__ == "__main__":
demo()
+252 -83
View File
@@ -12,18 +12,6 @@ from urllib.parse import quote, unquote, urlparse
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("ftp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool: def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool): if isinstance(value, bool):
return value return value
@@ -155,20 +143,54 @@ class FTP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None): def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config) super().__init__(config)
conf = _pick_provider_config(self.config) _instance_name, conf = self.resolve_plugin_instance()
self._host = str(conf.get("host") or "").strip() defaults = self._settings_from_config(conf)
self._tls = _coerce_bool(conf.get("tls"), False) self._host = str(defaults.get("host") or "").strip()
self._port = _coerce_int(conf.get("port"), 21) self._tls = bool(defaults.get("tls"))
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous" self._port = int(defaults.get("port") or 21)
password_value = conf.get("password") self._username = str(defaults.get("username") or "anonymous").strip() or "anonymous"
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@" self._password = str(defaults.get("password") or "anonymous@").strip() or "anonymous@"
self._passive = _coerce_bool(conf.get("passive"), True) self._passive = bool(defaults.get("passive"))
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
password_value = entry.get("password")
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"tls": _coerce_bool(entry.get("tls"), False),
"port": _coerce_int(entry.get("port"), 21),
"username": str(entry.get("username") or entry.get("user") or "anonymous").strip() or "anonymous",
"password": str(password_value).strip() if password_value not in (None, "") else "anonymous@",
"passive": _coerce_bool(entry.get("passive"), True),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool: def validate(self) -> bool:
return bool(self._host) settings = self._resolve_settings()
return bool(settings.get("host"))
def config_helper_text(self) -> str: def config_helper_text(self) -> str:
return "Test the configured FTP/FTPS settings before searching or uploading." return "Test the configured FTP/FTPS settings before searching or uploading."
@@ -186,13 +208,14 @@ class FTP(Provider):
if str(action_id or "").strip().lower() != "test_connection": if str(action_id or "").strip().lower() != "test_connection":
return super().run_config_action(action_id, **_kwargs) return super().run_config_action(action_id, **_kwargs)
if not self._host: settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the FTP connection."} return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
ftp = None ftp = None
try: try:
ftp = self._connect() ftp = self._connect(settings=settings)
active_path = self._base_path or "/" active_path = str(settings.get("base_path") or "/")
try: try:
ftp.cwd(active_path) ftp.cwd(active_path)
resolved_path = ftp.pwd() resolved_path = ftp.pwd()
@@ -200,7 +223,7 @@ class FTP(Provider):
resolved_path = active_path resolved_path = active_path
return { return {
"ok": True, "ok": True,
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.", "message": f"Connected to FTP {settings.get('host')}:{settings.get('port')} and reached {resolved_path}.",
} }
except Exception as exc: except Exception as exc:
return {"ok": False, "message": f"FTP connection failed: {exc}"} return {"ok": False, "message": f"FTP connection failed: {exc}"}
@@ -211,6 +234,10 @@ class FTP(Provider):
text, inline = parse_inline_query_arguments(query) text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {} filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"): if inline.get("path"):
filters["path"] = inline.get("path") filters["path"] = inline.get("path")
if inline.get("depth"): if inline.get("depth"):
@@ -221,17 +248,21 @@ class FTP(Provider):
return text, filters return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip() text = str(query or "").strip()
if not text or text == "*": if not text or text == "*":
return f"FTP: {active_path}" return f"FTP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"FTP: {text} @ {active_path}" return f"FTP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return { return {
"plugin": self.name, "plugin": self.name,
"host": self._host, "instance": settings.get("instance"),
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), "host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(), "query": str(query or "").strip(),
} }
@@ -244,15 +275,21 @@ class FTP(Provider):
) -> List[SearchResult]: ) -> List[SearchResult]:
_ = kwargs _ = kwargs
active_filters = dict(filters or {}) active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(filters=active_filters, require_explicit=True)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) if not settings.get("host"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower() type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip() needle = str(query or "").strip()
max_results = max(0, int(limit or 0)) max_results = max(0, int(limit or 0))
if max_results <= 0: if max_results <= 0:
return [] return []
ftp = self._connect() ftp = self._connect(settings=settings)
try: try:
return self._search_directory( return self._search_directory(
ftp, ftp,
@@ -261,6 +298,7 @@ class FTP(Provider):
limit=max_results, limit=max_results,
search_depth=search_depth, search_depth=search_depth,
type_filter=type_filter, type_filter=type_filter,
settings=settings,
) )
finally: finally:
self._close(ftp) self._close(ftp)
@@ -278,19 +316,23 @@ class FTP(Provider):
target_path = "" target_path = ""
target_title = "" target_title = ""
instance_name = ""
for item in selected_items or []: for item in selected_items or []:
metadata = self._item_metadata(item) metadata = self._item_metadata(item)
if not metadata.get("is_dir"): if not metadata.get("is_dir"):
continue continue
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path) settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip() target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path: if target_path:
break break
if not target_path: if not target_path:
return False return False
ftp = self._connect() settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ftp = self._connect(settings=settings)
try: try:
rows = self._search_directory( rows = self._search_directory(
ftp, ftp,
@@ -299,6 +341,7 @@ class FTP(Provider):
limit=500, limit=500,
search_depth=0, search_depth=0,
type_filter="any", type_filter="any",
settings=settings,
) )
finally: finally:
self._close(ftp) self._close(ftp)
@@ -310,18 +353,23 @@ class FTP(Provider):
return True return True
title = target_title or target_path title = target_title or target_path
table = Table(f"FTP: {title}")._perseverance(True) table = Table(f"FTP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("ftp") table.set_table("ftp")
try: try:
table.set_table_metadata({ table.set_table_metadata({
"provider": "ftp", "provider": "ftp",
"host": self._host, "instance": instance_name or None,
"host": settings.get("host"),
"path": target_path, "path": target_path,
"view": "directory", "view": "directory",
}) })
except Exception: except Exception:
pass pass
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"]) source_args = ["-plugin", "ftp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = [] payloads: List[Dict[str, Any]] = []
for row in rows: for row in rows:
@@ -329,7 +377,7 @@ class FTP(Provider):
payloads.append(row.to_dict()) payloads.append(row.to_dict())
try: try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path}) ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
except Exception: except Exception:
pass pass
@@ -342,6 +390,77 @@ class FTP(Provider):
return True return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "FTP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
ftp_url = str(metadata.get("ftp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("ftp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=ftp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Ftp Url": ftp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"FTP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "FTP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None) metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"): if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -349,10 +468,15 @@ class FTP(Provider):
target = str(getattr(result, "path", "") or "").strip() target = str(getattr(result, "path", "") or "").strip()
if not target: if not target:
return None return None
return self.download_url(target, output_dir, title=getattr(result, "title", None)) instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url) parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"] remote_path = settings["path"]
if not remote_path or remote_path == "/": if not remote_path or remote_path == "/":
return None return None
@@ -365,13 +489,7 @@ class FTP(Provider):
destination_dir.mkdir(parents=True, exist_ok=True) destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename) destination = _unique_path(destination_dir / filename)
ftp = self._connect( ftp = self._connect(settings=settings)
host=settings["host"],
port=settings["port"],
username=settings["username"],
password=settings["password"],
tls=settings["tls"],
)
try: try:
with destination.open("wb") as handle: with destination.open("wb") as handle:
ftp.retrbinary(f"RETR {remote_path}", handle.write) ftp.retrbinary(f"RETR {remote_path}", handle.write)
@@ -404,7 +522,12 @@ class FTP(Provider):
return None, None, None return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-")) temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None: if downloaded is None:
try: try:
temp_dir.rmdir() temp_dir.rmdir()
@@ -424,35 +547,57 @@ class FTP(Provider):
if not local_path.exists() or not local_path.is_file(): if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}") raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
raise RuntimeError("No configured FTP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name) remote_path = self._join_remote_path(remote_dir, remote_name)
ftp = self._connect() ftp = self._connect(settings=settings)
try: try:
self._ensure_directory(ftp, remote_dir) self._ensure_directory(ftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
with local_path.open("rb") as handle: with local_path.open("rb") as handle:
ftp.storbinary(f"STOR {remote_path}", handle) ftp.storbinary(f"STOR {remote_path}", handle)
finally: finally:
self._close(ftp) self._close(ftp)
return self._build_url(remote_path) return self._build_url(remote_path, settings=settings)
def _connect( def _connect(
self, self,
*, *,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None, host: Optional[str] = None,
port: Optional[int] = None, port: Optional[int] = None,
username: Optional[str] = None, username: Optional[str] = None,
password: Optional[str] = None, password: Optional[str] = None,
tls: Optional[bool] = None, tls: Optional[bool] = None,
) -> ftplib.FTP: ) -> ftplib.FTP:
use_tls = self._tls if tls is None else bool(tls) resolved = dict(settings or {})
use_tls = bool(resolved.get("tls")) if tls is None else bool(tls)
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP() ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout) ftp.connect(
ftp.login(username or self._username, password or self._password) host or str(resolved.get("host") or self._host),
int(port or resolved.get("port") or self._port),
timeout=max(1, int(resolved.get("timeout") or self._timeout)),
)
ftp.login(
username or str(resolved.get("username") or self._username),
password or str(resolved.get("password") or self._password),
)
try: try:
ftp.set_pasv(self._passive) ftp.set_pasv(bool(resolved.get("passive")) if "passive" in resolved else self._passive)
except Exception: except Exception:
pass pass
if use_tls and isinstance(ftp, ftplib.FTP_TLS): if use_tls and isinstance(ftp, ftplib.FTP_TLS):
@@ -497,32 +642,39 @@ class FTP(Provider):
self, self,
remote_path: Any, remote_path: Any,
*, *,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None, host: Optional[str] = None,
port: Optional[int] = None, port: Optional[int] = None,
tls: Optional[bool] = None, tls: Optional[bool] = None,
) -> str: ) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/") path_text = self._normalize_remote_path(remote_path, default="/")
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp" scheme = "ftps" if ((bool(resolved.get("tls")) if tls is None else bool(tls))) else "ftp"
host_text = str(host or self._host).strip() host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or self._port) port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 21 else "" port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip()) parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "ftp").strip().lower() scheme = (parsed.scheme or "ftp").strip().lower()
host = parsed.hostname or self._host host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or self._port port = parsed.port or settings.get("port") or self._port
username = parsed.username or self._username username = parsed.username or settings.get("username") or self._username
password = parsed.password or self._password password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return { return {
"instance": settings.get("instance"),
"tls": scheme == "ftps", "tls": scheme == "ftps",
"host": host, "host": host,
"port": port, "port": port,
"username": username, "username": username,
"password": password, "password": password,
"path": path_text, "path": path_text,
"passive": settings.get("passive", self._passive),
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
} }
def _search_directory( def _search_directory(
@@ -534,21 +686,22 @@ class FTP(Provider):
limit: int, limit: int,
search_depth: int, search_depth: int,
type_filter: str, type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]: ) -> List[SearchResult]:
results: List[SearchResult] = [] results: List[SearchResult] = []
visited: set[str] = set() visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None: def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path) normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit: if normalized in visited or len(results) >= limit:
return return
visited.add(normalized) visited.add(normalized)
for entry in self._list_directory(ftp, normalized): for entry in self._list_directory(ftp, normalized, base_path=str(settings.get("base_path") or self._base_path)):
if len(results) >= limit: if len(results) >= limit:
return return
if self._matches_entry(entry, needle=needle, type_filter=type_filter): if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry)) results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0: if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("ftp_path") or normalized), depth_left - 1) walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
@@ -578,16 +731,18 @@ class FTP(Provider):
return False return False
return True return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult: def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
ftp_path = str(entry.get("ftp_path") or "/") ftp_path = str(entry.get("ftp_path") or "/")
ftp_url = self._build_url(ftp_path) ftp_url = self._build_url(ftp_path, settings=settings)
is_dir = bool(entry.get("is_dir")) is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size") size_value = entry.get("size")
modified = str(entry.get("modified") or "") modified = str(entry.get("modified") or "")
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/" parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = { metadata = {
"provider": "ftp", "provider": "ftp",
"host": self._host, "instance": instance_name or None,
"host": settings.get("host"),
"ftp_path": ftp_path, "ftp_path": ftp_path,
"ftp_url": ftp_url, "ftp_url": ftp_url,
"selection_url": ftp_url, "selection_url": ftp_url,
@@ -599,6 +754,13 @@ class FTP(Provider):
if modified: if modified:
metadata["modified"] = modified metadata["modified"] = modified
selection_args = ["-url", ftp_url]
selection_action = ["download-file", "-plugin", "ftp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", ftp_url])
return SearchResult( return SearchResult(
table="ftp", table="ftp",
title=str(entry.get("name") or ftp_path), title=str(entry.get("name") or ftp_path),
@@ -615,13 +777,13 @@ class FTP(Provider):
("Size", "" if size_value is None else str(size_value)), ("Size", "" if size_value is None else str(size_value)),
("Modified", modified), ("Modified", modified),
], ],
selection_args=None if is_dir else ["-url", ftp_url], selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url], selection_action=None if is_dir else selection_action,
full_metadata=metadata, full_metadata=metadata,
) )
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]: def _list_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path) normalized = self._normalize_remote_path(remote_path, default=base_path)
try: try:
entries: List[Dict[str, Any]] = [] entries: List[Dict[str, Any]] = []
for name, facts in ftp.mlsd(normalized): for name, facts in ftp.mlsd(normalized):
@@ -716,8 +878,8 @@ class FTP(Provider):
return _format_timestamp(parts[1]) return _format_timestamp(parts[1])
return "" return ""
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None: def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path) normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/": if normalized == "/":
return return
partial = "" partial = ""
@@ -763,11 +925,18 @@ class FTP(Provider):
if path_text.startswith(("ftp://", "ftps://")): if path_text.startswith(("ftp://", "ftps://")):
ftp_path = self._normalize_remote_path(path_text, default=self._base_path) ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
if ftp_path: if ftp_path:
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path) base_path = str(metadata.get("base_path") or self._base_path)
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=base_path)
metadata.setdefault("selection_path", metadata["ftp_path"]) metadata.setdefault("selection_path", metadata["ftp_path"])
if metadata.get("ftp_path") and not metadata.get("ftp_url"): if metadata.get("ftp_path") and not metadata.get("ftp_url"):
metadata["ftp_url"] = self._build_url(metadata["ftp_path"]) metadata["ftp_url"] = self._build_url(
metadata["ftp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("ftp_url") and not metadata.get("selection_url"): if metadata.get("ftp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["ftp_url"] metadata["selection_url"] = metadata["ftp_url"]
+4 -4
View File
@@ -286,7 +286,7 @@ def _enrich_book_tags_from_isbn(isbn: str,
Priority: Priority:
1) OpenLibrary API-by-ISBN scrape (fast, structured) 1) OpenLibrary API-by-ISBN scrape (fast, structured)
2) isbnsearch.org scrape via MetadataProvider 2) isbnsearch.org scrape via metadata plugin
""" """
isbn_clean = re.sub(r"[^0-9Xx]", "", str(isbn or "")).upper() isbn_clean = re.sub(r"[^0-9Xx]", "", str(isbn or "")).upper()
@@ -381,11 +381,11 @@ def _enrich_book_tags_from_isbn(isbn: str,
except Exception: except Exception:
pass pass
# 2) isbnsearch metadata provider fallback. # 2) isbnsearch metadata plugin fallback.
try: try:
from plugins.metadata_provider import get_metadata_provider from plugins.metadata_provider import get_metadata_plugin
provider = get_metadata_provider("isbnsearch", provider = get_metadata_plugin("isbnsearch",
config or {}) config or {})
if provider is None: if provider is None:
return [], "" return [], ""
File diff suppressed because it is too large Load Diff
+248 -76
View File
@@ -16,18 +16,6 @@ from scp import SCPClient
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("scp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool: def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool): if isinstance(value, bool):
return value return value
@@ -166,20 +154,55 @@ class SCP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None): def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config) super().__init__(config)
conf = _pick_provider_config(self.config) _instance_name, conf = self.resolve_plugin_instance()
self._host = str(conf.get("host") or "").strip() defaults = self._settings_from_config(conf)
self._port = _coerce_int(conf.get("port"), 22) self._host = str(defaults.get("host") or "").strip()
self._username = str(conf.get("username") or conf.get("user") or "").strip() self._port = int(defaults.get("port") or 22)
self._password = str(conf.get("password") or "").strip() self._username = str(defaults.get("username") or "").strip()
self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip() self._password = str(defaults.get("password") or "").strip()
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) self._key_path = str(defaults.get("key_path") or "").strip()
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) self._timeout = max(1, int(defaults.get("timeout") or 20))
self._allow_agent = _coerce_bool(conf.get("allow_agent"), True) self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True) self._allow_agent = bool(defaults.get("allow_agent"))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") self._look_for_keys = bool(defaults.get("look_for_keys"))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"port": _coerce_int(entry.get("port"), 22),
"username": str(entry.get("username") or entry.get("user") or "").strip(),
"password": str(entry.get("password") or "").strip(),
"key_path": str(entry.get("key_path") or entry.get("identity_file") or "").strip(),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"allow_agent": _coerce_bool(entry.get("allow_agent"), True),
"look_for_keys": _coerce_bool(entry.get("look_for_keys"), True),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool: def validate(self) -> bool:
return bool(self._host and self._username) settings = self._resolve_settings()
return bool(settings.get("host") and settings.get("username"))
def config_helper_text(self) -> str: def config_helper_text(self) -> str:
return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here." return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here."
@@ -210,6 +233,10 @@ class SCP(Provider):
text, inline = parse_inline_query_arguments(query) text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {} filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"): if inline.get("path"):
filters["path"] = inline.get("path") filters["path"] = inline.get("path")
if inline.get("depth"): if inline.get("depth"):
@@ -220,17 +247,21 @@ class SCP(Provider):
return text, filters return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip() text = str(query or "").strip()
if not text or text == "*": if not text or text == "*":
return f"SCP: {active_path}" return f"SCP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"SCP: {text} @ {active_path}" return f"SCP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return { return {
"plugin": self.name, "plugin": self.name,
"host": self._host, "instance": settings.get("instance"),
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), "host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(), "query": str(query or "").strip(),
} }
@@ -243,15 +274,21 @@ class SCP(Provider):
) -> List[SearchResult]: ) -> List[SearchResult]:
_ = kwargs _ = kwargs
active_filters = dict(filters or {}) active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(filters=active_filters, require_explicit=True)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) if not settings.get("host") or not settings.get("username"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower() type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip() needle = str(query or "").strip()
max_results = max(0, int(limit or 0)) max_results = max(0, int(limit or 0))
if max_results <= 0: if max_results <= 0:
return [] return []
ssh = self._connect_ssh() ssh = self._connect_ssh(settings)
sftp = None sftp = None
try: try:
try: try:
@@ -266,6 +303,7 @@ class SCP(Provider):
limit=max_results, limit=max_results,
search_depth=search_depth, search_depth=search_depth,
type_filter=type_filter, type_filter=type_filter,
settings=settings,
) )
return self._search_directory( return self._search_directory(
@@ -275,6 +313,7 @@ class SCP(Provider):
limit=max_results, limit=max_results,
search_depth=search_depth, search_depth=search_depth,
type_filter=type_filter, type_filter=type_filter,
settings=settings,
) )
finally: finally:
self._close_client(sftp) self._close_client(sftp)
@@ -293,19 +332,23 @@ class SCP(Provider):
target_path = "" target_path = ""
target_title = "" target_title = ""
instance_name = ""
for item in selected_items or []: for item in selected_items or []:
metadata = self._item_metadata(item) metadata = self._item_metadata(item)
if not metadata.get("is_dir"): if not metadata.get("is_dir"):
continue continue
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path) settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip() target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path: if target_path:
break break
if not target_path: if not target_path:
return False return False
ssh = self._connect_ssh() settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ssh = self._connect_ssh(settings)
sftp = None sftp = None
try: try:
try: try:
@@ -320,6 +363,7 @@ class SCP(Provider):
limit=500, limit=500,
search_depth=0, search_depth=0,
type_filter="any", type_filter="any",
settings=settings,
) )
else: else:
rows = self._search_directory( rows = self._search_directory(
@@ -329,6 +373,7 @@ class SCP(Provider):
limit=500, limit=500,
search_depth=0, search_depth=0,
type_filter="any", type_filter="any",
settings=settings,
) )
finally: finally:
self._close_client(sftp) self._close_client(sftp)
@@ -341,18 +386,23 @@ class SCP(Provider):
return True return True
title = target_title or target_path title = target_title or target_path
table = Table(f"SCP: {title}")._perseverance(True) table = Table(f"SCP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("scp") table.set_table("scp")
try: try:
table.set_table_metadata({ table.set_table_metadata({
"provider": "scp", "provider": "scp",
"host": self._host, "instance": instance_name or None,
"host": settings.get("host"),
"path": target_path, "path": target_path,
"view": "directory", "view": "directory",
}) })
except Exception: except Exception:
pass pass
table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"]) source_args = ["-plugin", "scp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = [] payloads: List[Dict[str, Any]] = []
for row in rows: for row in rows:
@@ -360,7 +410,7 @@ class SCP(Provider):
payloads.append(row.to_dict()) payloads.append(row.to_dict())
try: try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path}) ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
except Exception: except Exception:
pass pass
@@ -373,6 +423,77 @@ class SCP(Provider):
return True return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "SCP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
scp_url = str(metadata.get("scp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("scp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=scp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Scp Url": scp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"SCP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "SCP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None) metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"): if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -380,10 +501,15 @@ class SCP(Provider):
target = str(getattr(result, "path", "") or "").strip() target = str(getattr(result, "path", "") or "").strip()
if not target: if not target:
return None return None
return self.download_url(target, output_dir, title=getattr(result, "title", None)) instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url) parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"] remote_path = settings["path"]
if not remote_path or remote_path == "/": if not remote_path or remote_path == "/":
return None return None
@@ -431,7 +557,12 @@ class SCP(Provider):
return None, None, None return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-")) temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None: if downloaded is None:
try: try:
temp_dir.rmdir() temp_dir.rmdir()
@@ -451,11 +582,24 @@ class SCP(Provider):
if not local_path.exists() or not local_path.is_file(): if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}") raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host") or not settings.get("username"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
raise RuntimeError("No configured SCP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name) remote_path = self._join_remote_path(remote_dir, remote_name)
ssh = self._connect_ssh() ssh = self._connect_ssh(settings)
sftp = None sftp = None
scp_client = None scp_client = None
try: try:
@@ -466,7 +610,7 @@ class SCP(Provider):
raise raise
self._ensure_directory_via_ssh(ssh, remote_dir) self._ensure_directory_via_ssh(ssh, remote_dir)
else: else:
self._ensure_directory(sftp, remote_dir) self._ensure_directory(sftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
scp_client = self._open_scp(ssh) scp_client = self._open_scp(ssh)
scp_client.put(str(local_path), remote_path=remote_path) scp_client.put(str(local_path), remote_path=remote_path)
finally: finally:
@@ -474,19 +618,20 @@ class SCP(Provider):
self._close_client(sftp) self._close_client(sftp)
self._close_client(ssh) self._close_client(ssh)
return self._build_url(remote_path) return self._build_url(remote_path, settings=settings)
def _run_test_connection(self) -> Dict[str, Any]: def _run_test_connection(self) -> Dict[str, Any]:
if not self._host: settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the SCP connection."} return {"ok": False, "message": "Set 'host' before testing the SCP connection."}
if not self._username: if not settings.get("username"):
return {"ok": False, "message": "Set 'username' before testing the SCP connection."} return {"ok": False, "message": "Set 'username' before testing the SCP connection."}
ssh = None ssh = None
sftp = None sftp = None
try: try:
ssh = self._connect_ssh() ssh = self._connect_ssh(settings)
base_path = self._base_path or "/" base_path = str(settings.get("base_path") or "/")
transport_detail = "SFTP available" transport_detail = "SFTP available"
try: try:
sftp = self._open_sftp(ssh) sftp = self._open_sftp(ssh)
@@ -502,10 +647,11 @@ class SCP(Provider):
except Exception: except Exception:
is_dir = False is_dir = False
detail = f" and confirmed {base_path}" if is_dir else "" detail = f" and confirmed {base_path}" if is_dir else ""
auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth" key_path = str(settings.get("key_path") or "").strip()
auth_mode = f"key {key_path}" if key_path else "password/agent auth"
return { return {
"ok": True, "ok": True,
"message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.", "message": f"Connected to SCP {settings.get('host')}:{settings.get('port')} as {settings.get('username')} via {auth_mode}. {transport_detail}{detail}.",
} }
except Exception as exc: except Exception as exc:
return {"ok": False, "message": f"SCP connection failed: {exc}"} return {"ok": False, "message": f"SCP connection failed: {exc}"}
@@ -514,7 +660,9 @@ class SCP(Provider):
self._close_client(ssh) self._close_client(ssh)
def _generate_ssh_keypair(self) -> Dict[str, Any]: def _generate_ssh_keypair(self) -> Dict[str, Any]:
target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa") settings = self._resolve_settings()
key_path = str(settings.get("key_path") or "").strip()
target = Path(key_path).expanduser() if key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
try: try:
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
except Exception as exc: except Exception as exc:
@@ -530,7 +678,7 @@ class SCP(Provider):
try: try:
key = paramiko.RSAKey.generate(bits=4096) key = paramiko.RSAKey.generate(bits=4096)
key.write_private_key_file(str(target)) key.write_private_key_file(str(target))
comment = f"{self._username or 'medeia'}@{self._host or 'scp'}" comment = f"{settings.get('username') or 'medeia'}@{settings.get('host') or 'scp'}"
public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8") public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8")
try: try:
target.chmod(0o600) target.chmod(0o600)
@@ -654,34 +802,40 @@ class SCP(Provider):
self, self,
remote_path: Any, remote_path: Any,
*, *,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None, host: Optional[str] = None,
port: Optional[int] = None, port: Optional[int] = None,
scheme: str = "scp", scheme: str = "scp",
) -> str: ) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/") path_text = self._normalize_remote_path(remote_path, default="/")
host_text = str(host or self._host).strip() host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or self._port) port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 22 else "" port_suffix = f":{port_value}" if port_value and port_value != 22 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip()) parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "scp").strip().lower() scheme = (parsed.scheme or "scp").strip().lower()
host = parsed.hostname or self._host host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or self._port port = parsed.port or settings.get("port") or self._port
username = parsed.username or self._username username = parsed.username or settings.get("username") or self._username
password = parsed.password or self._password password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return { return {
"instance": settings.get("instance"),
"scheme": scheme, "scheme": scheme,
"host": host, "host": host,
"port": port, "port": port,
"username": username, "username": username,
"password": password, "password": password,
"key_path": self._key_path, "key_path": settings.get("key_path") or self._key_path,
"allow_agent": self._allow_agent, "allow_agent": settings.get("allow_agent", self._allow_agent),
"look_for_keys": self._look_for_keys, "look_for_keys": settings.get("look_for_keys", self._look_for_keys),
"path": path_text, "path": path_text,
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
} }
def _search_directory( def _search_directory(
@@ -693,12 +847,13 @@ class SCP(Provider):
limit: int, limit: int,
search_depth: int, search_depth: int,
type_filter: str, type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]: ) -> List[SearchResult]:
results: List[SearchResult] = [] results: List[SearchResult] = []
visited: set[str] = set() visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None: def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path) normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit: if normalized in visited or len(results) >= limit:
return return
visited.add(normalized) visited.add(normalized)
@@ -707,7 +862,7 @@ class SCP(Provider):
if len(results) >= limit: if len(results) >= limit:
return return
if self._matches_entry(entry, needle=needle, type_filter=type_filter): if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry)) results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0: if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("scp_path") or normalized), depth_left - 1) walk(str(entry.get("scp_path") or normalized), depth_left - 1)
@@ -723,6 +878,7 @@ class SCP(Provider):
limit: int, limit: int,
search_depth: int, search_depth: int,
type_filter: str, type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]: ) -> List[SearchResult]:
entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth) entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth)
results: List[SearchResult] = [] results: List[SearchResult] = []
@@ -730,7 +886,7 @@ class SCP(Provider):
if len(results) >= limit: if len(results) >= limit:
break break
if self._matches_entry(entry, needle=needle, type_filter=type_filter): if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry)) results.append(self._build_result(entry, settings=settings))
return results return results
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool: def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
@@ -756,16 +912,18 @@ class SCP(Provider):
return False return False
return True return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult: def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
scp_path = str(entry.get("scp_path") or "/") scp_path = str(entry.get("scp_path") or "/")
scp_url = self._build_url(scp_path) scp_url = self._build_url(scp_path, settings=settings)
is_dir = bool(entry.get("is_dir")) is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size") size_value = entry.get("size")
modified = str(entry.get("modified") or "") modified = str(entry.get("modified") or "")
parent = posixpath.dirname(scp_path.rstrip("/")) or "/" parent = posixpath.dirname(scp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = { metadata = {
"provider": "scp", "provider": "scp",
"host": self._host, "instance": instance_name or None,
"host": settings.get("host"),
"scp_path": scp_path, "scp_path": scp_path,
"scp_url": scp_url, "scp_url": scp_url,
"selection_url": scp_url, "selection_url": scp_url,
@@ -777,6 +935,13 @@ class SCP(Provider):
if modified: if modified:
metadata["modified"] = modified metadata["modified"] = modified
selection_args = ["-url", scp_url]
selection_action = ["download-file", "-plugin", "scp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", scp_url])
return SearchResult( return SearchResult(
table="scp", table="scp",
title=str(entry.get("name") or scp_path), title=str(entry.get("name") or scp_path),
@@ -793,8 +958,8 @@ class SCP(Provider):
("Size", "" if size_value is None else str(size_value)), ("Size", "" if size_value is None else str(size_value)),
("Modified", modified), ("Modified", modified),
], ],
selection_args=None if is_dir else ["-url", scp_url], selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url], selection_action=None if is_dir else selection_action,
full_metadata=metadata, full_metadata=metadata,
) )
@@ -867,8 +1032,8 @@ class SCP(Provider):
) )
return entries return entries
def _ensure_directory(self, sftp: Any, remote_path: str) -> None: def _ensure_directory(self, sftp: Any, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path) normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/": if normalized == "/":
return return
partial = "" partial = ""
@@ -923,11 +1088,18 @@ class SCP(Provider):
if path_text.startswith(("scp://", "sftp://")): if path_text.startswith(("scp://", "sftp://")):
scp_path = self._normalize_remote_path(path_text, default=self._base_path) scp_path = self._normalize_remote_path(path_text, default=self._base_path)
if scp_path: if scp_path:
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path) base_path = str(metadata.get("base_path") or self._base_path)
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=base_path)
metadata.setdefault("selection_path", metadata["scp_path"]) metadata.setdefault("selection_path", metadata["scp_path"])
if metadata.get("scp_path") and not metadata.get("scp_url"): if metadata.get("scp_path") and not metadata.get("scp_url"):
metadata["scp_url"] = self._build_url(metadata["scp_path"]) metadata["scp_url"] = self._build_url(
metadata["scp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("scp_url") and not metadata.get("selection_url"): if metadata.get("scp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["scp_url"] metadata["selection_url"] = metadata["scp_url"]
+340 -4
View File
@@ -1,7 +1,343 @@
"""Plugin-namespace import shim for Tidal/HIFI manifest helpers. """Tidal/HIFI manifest helpers.
The implementation currently lives in ``Provider.tidal_manifest`` while the This module intentionally lives with the provider code (not cmdlets).
legacy namespace is phased out. New imports should prefer ``plugins``. It contains best-effort helpers for turning proxy-provided Tidal "manifest"
values into a playable input reference:
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
""" """
from Provider.tidal_manifest import * # noqa: F401,F403 from __future__ import annotations
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
_DEFAULT_TIDAL_TRACK_API_BASES = (
"https://triton.squid.wtf",
"https://wolf.qqdl.site",
"https://maus.qqdl.site",
"https://vogel.qqdl.site",
"https://katze.qqdl.site",
"https://hund.qqdl.site",
"https://tidal.kinoplus.online",
"https://tidal-api.binimum.org",
)
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _normalize_api_base(candidate: Any) -> Optional[str]:
text = str(candidate or "").strip()
if not text:
return None
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
return None
return text.rstrip("/")
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
bases: list[str] = []
seen: set[str] = set()
dynamic_candidates = [
metadata.get("_tidal_api_base"),
metadata.get("_api_base"),
metadata.get("api_base"),
metadata.get("base_url"),
]
for candidate in dynamic_candidates:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
return bases
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
except Exception:
return
attempted = False
for base in _iter_track_api_bases(metadata):
attempted = True
track_data: Optional[Dict[str, Any]] = None
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
try:
resp = client.get(
f"{base}/track/",
params=params,
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
track_data = data
break
except Exception:
continue
if isinstance(track_data, dict) and track_data:
try:
metadata.update(track_data)
except Exception:
pass
if not metadata.get("manifest") or not metadata.get("url"):
try:
resp_info = client.get(
f"{base}/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for key, value in info_data.items():
if key not in metadata or not metadata.get(key):
metadata[key] = value
except Exception:
pass
except Exception:
pass
if metadata.get("manifest"):
break
if attempted:
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None