goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updating and refining plugin system refactor

Browse Source
This commit is contained in:
Nose
2026-04-28 22:20:54 -07:00
parent 8685fbb723
commit 323c24f4f4
33 changed files with 4287 additions and 3312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
API/HydrusNetwork.py
+2 -33
View File
@@ -1,37 +1,6 @@
"""Hydrus API helpers and export utilities."""
"""Compatibility shim for the Hydrus plugin-owned API module."""
from __future__ import annotations
import base64
import http.client
import json
import os
import re
import shutil
import subprocess
import sys
import time
from collections import deque
from SYS.logger import log
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS as GLOBAL_SUPPORTED_EXTENSIONS
import tempfile
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast
from urllib.parse import urlsplit, urlencode, quote, urlunsplit, unquote
import httpx
logger = logging.getLogger(__name__)
from SYS.utils import (
decode_cbor,
jsonify,
ensure_directory,
unique_path,
)
from .HTTP import HTTPClient
from plugins.hydrusnetwork.api import * # noqa: F401,F403
class HydrusRequestError(RuntimeError):
CLI.py
+115 -26
View File
@@ -506,32 +506,28 @@ class CmdletIntrospection:
if normalized_arg == "plugin":
canonical_cmd = (cmd_name or "").replace("_", "-").lower()
try:
from ProviderCore.registry import (
list_search_plugin_names,
list_upload_plugin_names,
)
from ProviderCore.registry import list_plugin_names_with_capability
except Exception:
list_search_plugin_names = None # type: ignore
list_upload_plugin_names = None # type: ignore
list_plugin_names_with_capability = None # type: ignore
provider_choices: List[str] = []
plugin_choices: List[str] = []
if canonical_cmd in {"add-file"} and list_upload_plugin_names is not None:
return list_upload_plugin_names() or []
if canonical_cmd in {"add-file"} and list_plugin_names_with_capability is not None:
return list_plugin_names_with_capability("upload") or []
if list_search_plugin_names is not None:
provider_choices = list_search_plugin_names() or []
if list_plugin_names_with_capability is not None:
plugin_choices = list_plugin_names_with_capability("search") or []
if provider_choices:
return provider_choices
if plugin_choices:
return plugin_choices
if normalized_arg == "scrape":
try:
from plugins.metadata_provider import list_metadata_providers
from plugins.metadata_provider import list_metadata_plugins
meta_providers = list_metadata_providers(config) or {}
if meta_providers:
return sorted(meta_providers.keys())
metadata_plugins = list_metadata_plugins(config) or {}
if metadata_plugins:
return sorted(metadata_plugins.keys())
except Exception:
pass
@@ -704,6 +700,77 @@ class CmdletCompleter(Completer):
return tokens[idx + 1]
return None
@staticmethod
def _selected_plugin_name(cmd_name: str, stage_tokens: Sequence[str]) -> Optional[str]:
canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower()
if canonical_cmd not in {"search-file", "add-file", "download-file"}:
return None
return CmdletCompleter._flag_value(stage_tokens, "-plugin", "--plugin")
@staticmethod
def _plugin_instance_choices(plugin_name: Optional[str], config: Dict[str, Any]) -> List[str]:
plugin_key = str(plugin_name or "").strip().lower()
if not plugin_key:
return []
try:
from ProviderCore.registry import get_plugin_class
except Exception:
return []
plugin_class = get_plugin_class(plugin_key)
if plugin_class is None:
return []
try:
plugin = plugin_class(config)
except Exception:
return []
try:
instances = plugin.configured_instances()
except Exception:
return []
out: List[str] = []
seen: Set[str] = set()
for value in instances or []:
text = str(value or "").strip()
lowered = text.lower()
if not text or lowered in seen:
continue
seen.add(lowered)
out.append(text)
return out
def _filter_stage_arg_names(
self,
*,
cmd_name: str,
stage_tokens: Sequence[str],
config: Dict[str, Any],
arg_names: List[str],
) -> List[str]:
if not arg_names:
return []
canonical_cmd = str(cmd_name or "").replace("_", "-").strip().lower()
plugin_name = self._selected_plugin_name(canonical_cmd, stage_tokens)
instance_choices = self._plugin_instance_choices(plugin_name, config)
has_named_instances = bool(instance_choices)
filtered: List[str] = []
for arg in arg_names:
logical = str(arg or "").lstrip("-").strip().lower()
if logical == "instance":
if not plugin_name or not has_named_instances:
continue
if canonical_cmd == "search-file" and logical == "open":
if str(plugin_name or "").strip().lower() != "alldebrid":
continue
filtered.append(arg)
return filtered
def get_completions(
self,
document: Document,
@@ -742,7 +809,12 @@ class CmdletCompleter(Completer):
if cmd_name not in self.cmdlet_names:
return
arg_names = self._cmdlet_args(cmd_name, config)
arg_names = self._filter_stage_arg_names(
cmd_name=cmd_name,
stage_tokens=stage_tokens,
config=config,
arg_names=self._cmdlet_args(cmd_name, config),
)
seen_logicals: Set[str] = set()
for arg in arg_names:
arg_low = arg.lower()
@@ -779,6 +851,8 @@ class CmdletCompleter(Completer):
if cmd_name == "search-file":
provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin")
selected_plugin = self._selected_plugin_name(cmd_name, stage_tokens)
query_specs = self._query_args(cmd_name, config)
query_flag_index = -1
for idx, tok in enumerate(stage_tokens):
@@ -886,28 +960,43 @@ class CmdletCompleter(Completer):
yield Completion(suggestion, start_position=start_pos)
return
choices = self._arg_choices(
cmd_name=cmd_name,
arg_name=prev_token,
config=config,
force=False,
)
normalized_prev = prev_token.lstrip("-").strip().lower()
choices: List[str] = []
if normalized_prev == "instance" and selected_plugin:
choices = self._plugin_instance_choices(selected_plugin, config)
if not choices:
choices = self._arg_choices(
cmd_name=cmd_name,
arg_name=prev_token,
config=config,
force=False,
)
if choices:
choice_list = choices
normalized_prev = prev_token.lstrip("-").strip().lower()
if normalized_prev in {"plugin", "provider"} and current_token:
current_lower = current_token.lower()
filtered = [c for c in choices if current_lower in c.lower()]
if filtered:
choice_list = filtered
if normalized_prev == "instance" and current_token:
current_lower = current_token.lower()
filtered = [c for c in choice_list if current_lower in c.lower()]
if filtered:
choice_list = filtered
for choice in choice_list:
yield Completion(choice, start_position=-len(current_token))
# Example: if the user has typed `download-file -url ...`, then `url`
# is considered used and should not be suggested again (even as `--url`).
return
arg_names = self._cmdlet_args(cmd_name, config)
arg_names = self._filter_stage_arg_names(
cmd_name=cmd_name,
stage_tokens=stage_tokens,
config=config,
arg_names=self._cmdlet_args(cmd_name, config),
)
used_logicals = self._used_arg_logicals(cmd_name, stage_tokens, config)
logical_seen: Set[str] = set()
for arg in arg_names:
MPV/LUA/main.lua
+6
View File
@@ -4105,6 +4105,9 @@ function M._apply_web_subtitle_load_defaults(reason)
if target == '' or not _is_http_url(target) then
return false
end
if not _is_ytdlp_url(target) then
return false
end
M._prepare_ytdl_format_for_web_load(target, reason or 'on-load')
@@ -4250,6 +4253,9 @@ function M._ensure_current_subtitles_visible(reason)
if (not current or current == '') and (path == '' or not _is_http_url(path)) then
return false
end
if (not current or current == '') and (path == '' or not _is_ytdlp_url(path)) then
return false
end
local track_id, source, already_selected = M._find_subtitle_track_candidate()
if not track_id then
Provider/example_provider.py
+5 -256
View File
@@ -1,259 +1,8 @@
"""Example plugin that uses the new `ResultTable` API.
"""Legacy compatibility shim for the strict adapter example module.
This module demonstrates a minimal provider adapter that yields `ResultModel`
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
(`render_table`) for demonstration.
Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default
The active implementation now lives in ``plugins.example_provider`` so the
plugin namespace owns the example adapter module. Keep this file only to avoid
breaking old imports while the legacy ``Provider`` package is phased out.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column
SAMPLE_ITEMS = [
{
"name": "Book of Awe.pdf",
"path": "sample/Book of Awe.pdf",
"ext": "pdf",
"size": 1024000,
"source": "example",
},
{
"name": "Song of Joy.mp3",
"path": "sample/Song of Joy.mp3",
"ext": "mp3",
"size": 5120000,
"source": "example",
},
{
"name": "Cover Image.jpg",
"path": "sample/Cover Image.jpg",
"ext": "jpg",
"size": 20480,
"source": "example",
},
]
def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]:
"""Convert provider-specific items into `ResultModel` instances.
This adapter enforces the strict API requirement: it yields only
`ResultModel` instances (no legacy dict objects).
"""
for it in items:
title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "")
yield ResultModel(
title=str(title),
path=str(it.get("path")) if it.get("path") else None,
ext=str(it.get("ext")) if it.get("ext") else None,
size_bytes=int(it.get("size")) if it.get("size") is not None else None,
metadata=dict(it),
source=str(it.get("source")) if it.get("source") else "example",
)
# Columns are intentionally *not* mandated. Create a factory that inspects
# sample rows and builds only columns that make sense for the provider data.
from SYS.result_table_api import metadata_column
def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]:
cols: List[ColumnSpec] = [title_column()]
# If any row has an extension, include Ext column
if any(getattr(r, "ext", None) for r in rows):
cols.append(ext_column())
# If any row has size, include Size column
if any(getattr(r, "size_bytes", None) for r in rows):
cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v)))
# Add any top-level metadata keys discovered (up to 3) as optional columns
seen_keys = []
for r in rows:
for k in (r.metadata or {}).keys():
if k in ("name", "title", "path"):
continue
if k not in seen_keys:
seen_keys.append(k)
if len(seen_keys) >= 3:
break
if len(seen_keys) >= 3:
break
for k in seen_keys:
cols.append(metadata_column(k))
return cols
# Selection function: cmdlets rely on this to build selector args when the user
# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer
# -path if available, otherwise fall back to -title.
def selection_fn(row: ResultModel) -> List[str]:
if row.path:
return ["-path", row.path]
return ["-title", row.title]
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
selection_fn=selection_fn,
metadata={"description": "Example provider demonstrating dynamic columns and selectors"},
)
def _format_size(size: Any) -> str:
try:
s = int(size)
except Exception:
return ""
if s >= 1024 ** 3:
return f"{s / (1024 ** 3):.2f} GB"
if s >= 1024 ** 2:
return f"{s / (1024 ** 2):.2f} MB"
if s >= 1024:
return f"{s / 1024:.2f} KB"
return f"{s} B"
def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str:
"""Render a simple ASCII table of `rows` using `columns`.
This is intentionally very small and dependency-free for demonstration.
Renderers in the project should implement the `Renderer` protocol.
"""
rows = list(rows)
# Build cell matrix (strings)
matrix: List[List[str]] = []
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
matrix.append(cells)
# Compute column widths as max(header, content)
headers = [c.header for c in columns]
widths = [len(h) for h in headers]
for row_cells in matrix:
for i, cell in enumerate(row_cells):
widths[i] = max(widths[i], len(cell))
# Helper to format a row
def fmt_row(cells: List[str]) -> str:
return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
lines: List[str] = []
lines.append(fmt_row(headers))
lines.append("-+-".join("-" * w for w in widths))
for row_cells in matrix:
lines.append(fmt_row(row_cells))
return "\n".join(lines)
# Rich-based renderer (returns a Rich Table renderable)
def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]):
"""Render rows as a `rich.table.Table` for terminal output.
Returns the Table object; callers may `Console.print(table)` to render.
"""
try:
from rich.table import Table as RichTable
except Exception as exc: # pragma: no cover - rare if rich missing
raise RuntimeError("rich is required for rich renderer") from exc
table = RichTable(show_header=True, header_style="bold")
for col in columns:
table.add_column(col.header)
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
table.add_row(*cells)
return table
def demo() -> None:
rows = list(adapter(SAMPLE_ITEMS))
table = render_table_rich(rows, columns_factory(rows))
try:
from rich.console import Console
except Exception:
# Fall back to plain printing if rich is not available
print("Example provider output:")
print(render_table(rows, columns_factory(rows)))
return
console = Console()
console.print("Example provider output:")
console.print(table)
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_plugin
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
# Render
try:
from rich.console import Console
except Exception:
print(render_table(rows, cols))
sel_args = provider.selection_args(rows[idx])
print("Selection args for row", idx, "->", sel_args)
return
console = Console()
console.print("Example provider output:")
console.print(render_table_rich(rows, cols))
# Selection args example
sel = provider.selection_args(rows[idx])
console.print("Selection args for row", idx, "->", sel)
if __name__ == "__main__":
demo()
from plugins.example_provider import * # noqa: F401,F403
Provider/metadata_provider.py
+6 -2024
View File
File diff suppressed because it is too large Load Diff
Provider/tidal_manifest.py
+5 -340
View File
@@ -1,343 +1,8 @@
"""Tidal/HIFI manifest helpers.
"""Legacy compatibility shim for Tidal manifest helpers.
This module intentionally lives with the provider code (not cmdlets).
It contains best-effort helpers for turning proxy-provided Tidal "manifest"
values into a playable input reference:
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
The active implementation now lives in ``plugins.tidal_manifest`` so the
plugin namespace owns the manifest helper module. Keep this file only to avoid
breaking old imports while the legacy ``Provider`` package is phased out.
"""
from __future__ import annotations
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
_DEFAULT_TIDAL_TRACK_API_BASES = (
"https://triton.squid.wtf",
"https://wolf.qqdl.site",
"https://maus.qqdl.site",
"https://vogel.qqdl.site",
"https://katze.qqdl.site",
"https://hund.qqdl.site",
"https://tidal.kinoplus.online",
"https://tidal-api.binimum.org",
)
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _normalize_api_base(candidate: Any) -> Optional[str]:
text = str(candidate or "").strip()
if not text:
return None
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
return None
return text.rstrip("/")
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
bases: list[str] = []
seen: set[str] = set()
dynamic_candidates = [
metadata.get("_tidal_api_base"),
metadata.get("_api_base"),
metadata.get("api_base"),
metadata.get("base_url"),
]
for candidate in dynamic_candidates:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
return bases
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
except Exception:
return
attempted = False
for base in _iter_track_api_bases(metadata):
attempted = True
track_data: Optional[Dict[str, Any]] = None
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
try:
resp = client.get(
f"{base}/track/",
params=params,
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
track_data = data
break
except Exception:
continue
if isinstance(track_data, dict) and track_data:
try:
metadata.update(track_data)
except Exception:
pass
if not metadata.get("manifest") or not metadata.get("url"):
try:
resp_info = client.get(
f"{base}/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for key, value in info_data.items():
if key not in metadata or not metadata.get(key):
metadata[key] = value
except Exception:
pass
except Exception:
pass
if metadata.get("manifest"):
break
if attempted:
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None
from plugins.tidal_manifest import * # noqa: F401,F403
ProviderCore/base.py
+232 -33
View File
@@ -114,7 +114,7 @@ def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]:
class Provider(ABC):
"""Unified plugin base class.
This replaces the older split between search and upload providers.
This replaces the older split between search and upload plugins.
Concrete plugins may implement any subset of:
- search(query, ...)
- download(result, output_dir)
@@ -127,8 +127,8 @@ class Provider(ABC):
PLUGIN_NAME: str = ""
PLUGIN_ALIASES: Sequence[str] = ()
# Optional provider-driven defaults for what to do when a user selects @N from a
# provider table. The CLI uses this to auto-insert stages (e.g. download-file)
# Optional plugin-driven defaults for what to do when a user selects @N from a
# plugin table. The CLI uses this to auto-insert stages (e.g. download-file)
# without hardcoding table names.
#
# Example:
@@ -138,7 +138,7 @@ class Provider(ABC):
TABLE_AUTO_PREFIXES: Dict[str, Sequence[str]] = {}
AUTO_STAGE_USE_SELECTION_ARGS: bool = False
# Optional provider-declared configuration keys.
# Optional plugin-declared configuration keys.
# Used for dynamically generating config panels (e.g., missing credentials).
REQUIRED_CONFIG_KEYS: Sequence[str] = ()
@@ -176,7 +176,7 @@ class Provider(ABC):
return False
def get_table_type(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
"""Return the table type identifier for results from this provider."""
"""Return the table type identifier for results from this plugin."""
return self.name
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
@@ -197,12 +197,12 @@ class Provider(ABC):
@property
def prefers_transfer_progress(self) -> bool:
"""True if this provider prefers explicit transfer progress tracking (begin/finish) during download."""
"""True if this plugin prefers explicit transfer progress tracking (begin/finish) during download."""
return False
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
"""Return configuration schema for this provider.
"""Return configuration schema for this plugin.
Returns a list of dicts, each defining a field:
{
@@ -234,8 +234,124 @@ class Provider(ABC):
return []
return out
@classmethod
def plugin_config_key(cls) -> str:
return str(getattr(cls, "PLUGIN_NAME", None) or cls.__name__ or "").strip().lower()
@classmethod
def plugin_instance_filter_keys(cls) -> Tuple[str, ...]:
return ("instance", "store")
@classmethod
def plugin_config_field_keys(cls) -> set[str]:
keys: set[str] = set()
try:
for field in cls.config_schema() or []:
if not isinstance(field, dict):
continue
key = str(field.get("key") or "").strip().lower()
if key:
keys.add(key)
except Exception:
return set()
return keys
def requested_instance_name(
self,
filters: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Optional[str]:
for key in self.plugin_instance_filter_keys():
value = kwargs.get(key)
if value in (None, "") and isinstance(filters, dict):
value = filters.get(key)
text = str(value or "").strip()
if text:
return text
return None
def plugin_config_root(self) -> Dict[str, Any]:
if not isinstance(self.config, dict):
return {}
provider_cfg = self.config.get("provider")
if not isinstance(provider_cfg, dict):
return {}
entry = provider_cfg.get(self.plugin_config_key())
return dict(entry) if isinstance(entry, dict) else {}
def plugin_instance_configs(self) -> Dict[str, Dict[str, Any]]:
entry = self.plugin_config_root()
if not entry:
return {}
schema_keys = self.plugin_config_field_keys()
entry_keys = {str(key or "").strip().lower() for key in entry.keys()}
looks_like_single = bool(schema_keys and entry_keys.intersection(schema_keys))
if not looks_like_single and entry:
looks_like_single = not all(isinstance(value, dict) for value in entry.values())
if looks_like_single:
return {"default": dict(entry)}
instances: Dict[str, Dict[str, Any]] = {}
for raw_name, raw_value in entry.items():
if not isinstance(raw_value, dict):
continue
name = str(raw_name or "").strip()
if not name:
continue
instances[name] = dict(raw_value)
return instances
def configured_instances(self) -> List[str]:
instances = self.plugin_instance_configs()
if not instances:
return []
if set(instances.keys()) == {"default"}:
return []
return list(instances.keys())
def resolve_plugin_instance(
self,
instance_name: Optional[str] = None,
*,
require_explicit: bool = False,
) -> Tuple[Optional[str], Dict[str, Any]]:
instances = self.plugin_instance_configs()
if not instances:
return None, {}
requested = str(instance_name or "").strip()
if not requested:
first_name = next(iter(instances.keys()))
resolved = dict(instances[first_name])
if first_name != "default":
resolved.setdefault("_instance_name", first_name)
return (None if first_name == "default" else first_name), resolved
requested_lower = requested.lower()
for name, cfg in instances.items():
aliases = {str(name).strip().lower()}
explicit_name = str(cfg.get("_instance_name") or cfg.get("instance") or cfg.get("name") or "").strip().lower()
if explicit_name:
aliases.add(explicit_name)
if requested_lower in aliases:
resolved = dict(cfg)
if name != "default":
resolved.setdefault("_instance_name", name)
return (None if name == "default" else name), resolved
if require_explicit:
return None, {}
first_name = next(iter(instances.keys()))
resolved = dict(instances[first_name])
if first_name != "default":
resolved.setdefault("_instance_name", first_name)
return (None if first_name == "default" else first_name), resolved
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
"""Allow providers to normalize query text and parse inline arguments."""
"""Allow plugins to normalize query text and parse inline arguments."""
normalized = str(query or "").strip()
return normalized, {}
@@ -250,9 +366,9 @@ class Provider(ABC):
table_type: str = "",
table_meta: Optional[Dict[str, Any]] = None,
) -> Tuple[List[SearchResult], Optional[str], Optional[Dict[str, Any]]]:
"""Optional hook for provider-specific result transforms.
"""Optional hook for plugin-specific result transforms.
Cmdlets should avoid hardcoding provider quirks. Providers can override
Cmdlets should avoid hardcoding plugin quirks. Plugins can override
this to:
- expand/replace result sets (e.g., artist -> albums)
- override the table type
@@ -282,7 +398,7 @@ class Provider(ABC):
**kwargs: Any,
) -> List[SearchResult]:
"""Search for items matching the query."""
raise NotImplementedError(f"Provider '{self.name}' does not support search")
raise NotImplementedError(f"Plugin '{self.name}' does not support search")
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
"""Download an item from a search result."""
@@ -355,7 +471,7 @@ class Provider(ABC):
}
def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]:
"""Optional provider override to parse and act on URLs."""
"""Optional plugin override to parse and act on URLs."""
_ = url
_ = output_dir
@@ -428,24 +544,24 @@ class Provider(ABC):
return ""
def config_actions(self) -> List[Dict[str, Any]]:
"""Optional actions exposed in the config editor for this provider."""
"""Optional actions exposed in the config editor for this plugin."""
return []
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
"""Execute a provider-owned config action from the config editor."""
"""Execute a plugin-owned config action from the config editor."""
return {
"ok": False,
"message": f"Provider '{self.name}' does not support config action '{action_id}'.",
"message": f"Plugin '{self.name}' does not support config action '{action_id}'.",
}
def upload(self, file_path: str, **kwargs: Any) -> str:
"""Upload a file and return a URL or identifier."""
raise NotImplementedError(f"Provider '{self.name}' does not support upload")
raise NotImplementedError(f"Plugin '{self.name}' does not support upload")
def validate(self) -> bool:
"""Check if provider is available and properly configured."""
"""Check if the plugin is available and properly configured."""
return True
@@ -459,7 +575,7 @@ class Provider(ABC):
) -> bool:
"""Optional hook for handling `@N` selection semantics.
The CLI can delegate selection behavior to a provider/store instead of
The CLI can delegate selection behavior to a plugin/store instead of
applying the default selection filtering.
Return True if the selection was handled and default behavior should be skipped.
@@ -470,6 +586,101 @@ class Provider(ABC):
_ = stage_is_last
return False
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
"""Optionally render a terminal detail view for a selected plugin row."""
_selected_item, payload, _metadata = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
)
_ = table_type
_ = table_metadata
if not isinstance(payload, dict):
return False
detail_title = self.label
item_title = str(payload.get("title") or "").strip()
if item_title:
detail_title = f"{self.label}: {item_title}"
try:
from SYS.rich_display import render_item_details_panel
render_item_details_panel(payload, title=detail_title)
except Exception:
return False
try:
if hasattr(ctx, "set_last_result_items_only"):
ctx.set_last_result_items_only([payload])
except Exception:
pass
return True
def resolve_selection_detail_subject(
self,
selected_items: List[Any],
*,
stage_is_last: bool = True,
source_command: str = "",
require_media_kind: Optional[str] = None,
) -> Tuple[Optional[Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
"""Normalize a terminal `@N` selection into `(item, payload, metadata)`.
Custom plugin detail hooks can use this to share the common preconditions
for item panels instead of re-checking terminal/single-row/search-file
state in each plugin.
"""
if not stage_is_last or len(selected_items or []) != 1:
return None, None, None
normalized_source = str(source_command or "").replace("_", "-").strip().lower()
if normalized_source != "search-file":
return None, None, None
item: Any = selected_items[0]
payload: Optional[Dict[str, Any]]
if isinstance(item, dict):
payload = item
else:
payload = None
to_dict = getattr(item, "to_dict", None)
if callable(to_dict):
try:
maybe = to_dict()
except Exception:
maybe = None
if isinstance(maybe, dict):
payload = maybe
if not isinstance(payload, dict):
return item, None, None
meta: Dict[str, Any] = {}
nested = payload.get("full_metadata") or payload.get("metadata")
if isinstance(nested, dict):
meta = nested
if require_media_kind:
media_kind = str(payload.get("media_kind") or meta.get("media_kind") or "").strip().lower()
if media_kind != str(require_media_kind or "").strip().lower():
return item, None, None
return item, payload, meta
@classmethod
def selection_auto_stage(
cls,
@@ -478,10 +689,10 @@ class Provider(ABC):
) -> Optional[List[str]]:
"""Return a stage to auto-run after selecting from `table_type`.
This is used by the CLI to auto-insert default stages for provider tables
This is used by the CLI to auto-insert default stages for plugin tables
(e.g. select a YouTube row -> auto-run download-file).
Providers can implement this via class attributes (TABLE_AUTO_STAGES /
Plugins can implement this via class attributes (TABLE_AUTO_STAGES /
TABLE_AUTO_PREFIXES) or by overriding this method.
"""
t = str(table_type or "").strip().lower()
@@ -522,7 +733,7 @@ class Provider(ABC):
@classmethod
def url_patterns(cls) -> Tuple[str, ...]:
"""Return normalized URL patterns that this provider handles."""
"""Return normalized URL patterns that this plugin handles."""
patterns: List[str] = []
maybe_urls = getattr(cls, "URL", None)
if isinstance(maybe_urls, (list, tuple)):
@@ -564,15 +775,3 @@ class Provider(ABC):
return tuple(prefixes)
class SearchProvider(Provider):
"""Compatibility alias for older code.
Prefer inheriting from Provider directly.
"""
class FileProvider(Provider):
"""Compatibility alias for older code.
Prefer inheriting from Provider directly.
"""
ProviderCore/registry.py
+91 -100
View File
@@ -22,12 +22,24 @@ from urllib.parse import urlparse
from SYS.logger import log, debug
from ProviderCore.base import FileProvider, Provider, SearchProvider, SearchResult
from ProviderCore.base import Provider, SearchResult
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
def _class_supports_method(
plugin_class: Type[Provider],
method_name: str,
base_method: Any,
) -> bool:
try:
method = getattr(plugin_class, method_name, None)
except Exception:
return False
return callable(method) and method is not base_method
def _repo_root() -> Path:
try:
return Path(__file__).resolve().parents[1]
@@ -102,34 +114,34 @@ def _iter_external_plugin_entries(plugin_dir: Path) -> Iterable[Tuple[str, Path,
return tuple(out)
@dataclass(frozen=True)
class ProviderInfo:
class PluginInfo:
"""Metadata about a single plugin entry."""
canonical_name: str
provider_class: Type[Provider]
plugin_class: Type[Provider]
module: str
alias_names: Tuple[str, ...] = field(default_factory=tuple)
@property
def supports_search(self) -> bool:
return self.provider_class.search is not Provider.search
return _class_supports_method(self.plugin_class, "search", Provider.search)
@property
def supports_upload(self) -> bool:
try:
exposed = bool(getattr(self.provider_class, "EXPOSE_AS_FILE_PROVIDER", True))
exposed = bool(getattr(self.plugin_class, "EXPOSE_AS_FILE_PROVIDER", True))
except Exception:
exposed = True
return exposed and (self.provider_class.upload is not Provider.upload)
return exposed and _class_supports_method(self.plugin_class, "upload", Provider.upload)
class ProviderRegistry:
class PluginRegistry:
"""Handles discovery, registration, and lookup of built-in and external plugins."""
def __init__(self, package_name: str) -> None:
self.package_name = (package_name or "").strip()
self._infos: Dict[str, ProviderInfo] = {}
self._lookup: Dict[str, ProviderInfo] = {}
self._infos: Dict[str, PluginInfo] = {}
self._lookup: Dict[str, PluginInfo] = {}
self._modules: set[str] = set()
self._external_modules: set[str] = set()
self._builtin_package_dirs: Tuple[Path, ...] = ()
@@ -174,7 +186,7 @@ class ProviderRegistry:
return str(value or "").strip().lower()
def _candidate_names(self,
provider_class: Type[Provider],
plugin_class: Type[Provider],
override_name: Optional[str]) -> List[str]:
names: List[str] = []
seen: set[str] = set()
@@ -190,25 +202,25 @@ class ProviderRegistry:
if override_name:
_add(override_name)
else:
_add(getattr(provider_class, "PLUGIN_NAME", None))
_add(getattr(provider_class, "__name__", None))
_add(getattr(plugin_class, "PLUGIN_NAME", None))
_add(getattr(plugin_class, "__name__", None))
for alias in getattr(provider_class, "PLUGIN_ALIASES", ()) or ():
for alias in getattr(plugin_class, "PLUGIN_ALIASES", ()) or ():
_add(alias)
return names
def register(
self,
provider_class: Type[Provider],
plugin_class: Type[Provider],
*,
override_name: Optional[str] = None,
extra_aliases: Optional[Sequence[str]] = None,
module_name: Optional[str] = None,
replace: bool = False,
) -> ProviderInfo:
) -> PluginInfo:
"""Register a plugin class with canonical and alias names."""
candidates = self._candidate_names(provider_class, override_name)
candidates = self._candidate_names(plugin_class, override_name)
if not candidates:
raise ValueError("plugin name candidates are required")
@@ -233,10 +245,10 @@ class ProviderRegistry:
alias_seen.add(normalized)
alias_names.append(normalized)
info = ProviderInfo(
info = PluginInfo(
canonical_name=canonical,
provider_class=provider_class,
module=module_name or getattr(provider_class, "__module__", "") or "",
plugin_class=plugin_class,
module=module_name or getattr(plugin_class, "__module__", "") or "",
alias_names=tuple(alias_names),
)
@@ -261,7 +273,7 @@ class ProviderRegistry:
continue
if not issubclass(candidate, Provider):
continue
if candidate in {Provider, SearchProvider, FileProvider}:
if candidate is Provider:
continue
if getattr(candidate, "__module__", "") != module_name:
continue
@@ -311,7 +323,7 @@ class ProviderRegistry:
log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr)
def discover(self) -> None:
"""Import and register providers from the package."""
"""Import and register plugins from the package."""
if self._discovered or not self.package_name:
return
@@ -376,7 +388,7 @@ class ProviderRegistry:
self._sync_subclasses()
return
def get(self, name: str) -> Optional[ProviderInfo]:
def get(self, name: str) -> Optional[PluginInfo]:
if not name:
return None
@@ -398,7 +410,7 @@ class ProviderRegistry:
self.discover()
return self._lookup.get(normalized)
def iter_providers(self) -> Iterable[ProviderInfo]:
def iter_plugins(self) -> Iterable[PluginInfo]:
self.discover()
return tuple(self._infos.values())
@@ -406,12 +418,9 @@ class ProviderRegistry:
return self.get(name) is not None
def _sync_subclasses(self) -> None:
"""Walk all Provider subclasses in memory and register them."""
"""Walk all plugin subclasses in memory and register them."""
def _walk(cls: Type[Provider]) -> None:
for sub in cls.__subclasses__():
if sub in {SearchProvider, FileProvider}:
_walk(sub)
continue
try:
self.register(sub)
except Exception:
@@ -419,16 +428,14 @@ class ProviderRegistry:
_walk(sub)
_walk(Provider)
REGISTRY = ProviderRegistry("plugins")
REGISTRY = PluginRegistry("plugins")
PLUGIN_REGISTRY = REGISTRY
PluginInfo = ProviderInfo
PluginRegistry = ProviderRegistry
@lru_cache(maxsize=512)
def _provider_url_patterns(provider_class: Type[Provider]) -> Sequence[str]:
def _plugin_url_patterns(plugin_class: Type[Provider]) -> Sequence[str]:
try:
return list(provider_class.url_patterns())
return list(plugin_class.url_patterns())
except Exception:
return []
@@ -440,7 +447,7 @@ def register_plugin(
aliases: Optional[Sequence[str]] = None,
module_name: Optional[str] = None,
replace: bool = False,
) -> ProviderInfo:
) -> PluginInfo:
return REGISTRY.register(
plugin_class,
override_name=name,
@@ -454,7 +461,7 @@ def get_plugin_class(name: str) -> Optional[Type[Provider]]:
info = REGISTRY.get(name)
if info is None:
return None
return info.provider_class
return info.plugin_class
def selection_auto_stage_for_table(
@@ -481,7 +488,7 @@ def is_known_plugin_name(name: str) -> bool:
def _supports_search(provider: Provider) -> bool:
return provider.__class__.search is not Provider.search
return _class_supports_method(provider.__class__, "search", Provider.search)
def _supports_upload(provider: Provider) -> bool:
@@ -489,7 +496,25 @@ def _supports_upload(provider: Provider) -> bool:
exposed = bool(getattr(provider.__class__, "EXPOSE_AS_FILE_PROVIDER", True))
except Exception:
exposed = True
return exposed and (provider.__class__.upload is not Provider.upload)
return exposed and _class_supports_method(provider.__class__, "upload", Provider.upload)
def _supports_capability(provider: Provider, capability: str) -> bool:
capability_key = str(capability or "").strip().lower()
if capability_key == "search":
return _supports_search(provider)
if capability_key in {"upload", "file", "file-provider"}:
return _supports_upload(provider)
return False
def _info_supports_capability(info: PluginInfo, capability: str) -> bool:
capability_key = str(capability or "").strip().lower()
if capability_key == "search":
return bool(info.supports_search)
if capability_key in {"upload", "file", "file-provider"}:
return bool(info.supports_upload)
return False
def _normalize_choice_entry(entry: Any) -> Optional[Dict[str, Any]]:
@@ -555,7 +580,7 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
return None
try:
plugin = info.provider_class(config)
plugin = info.plugin_class(config)
if not plugin.validate():
debug(f"[plugin] Plugin '{name}' is not available")
return None
@@ -567,78 +592,50 @@ def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[P
def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
for info in REGISTRY.iter_plugins():
try:
plugin = info.provider_class(config)
plugin = info.plugin_class(config)
availability[info.canonical_name] = plugin.validate()
except Exception:
availability[info.canonical_name] = False
return availability
def get_search_plugin(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]:
def get_plugin_with_capability(
name: str,
capability: str,
config: Optional[Dict[str, Any]] = None,
) -> Optional[Provider]:
plugin = get_plugin(name, config)
if plugin is None:
return None
if not _supports_search(plugin):
debug(f"[plugin] Plugin '{name}' does not support search")
if not _supports_capability(plugin, capability):
debug(f"[plugin] Plugin '{name}' does not support capability '{capability}'")
return None
return plugin # type: ignore[return-value]
return plugin
def list_search_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
def list_plugins_with_capability(
capability: str,
config: Optional[Dict[str, Any]] = None,
) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
for info in REGISTRY.iter_plugins():
try:
plugin = info.provider_class(config)
plugin = info.plugin_class(config)
availability[info.canonical_name] = bool(
plugin.validate() and info.supports_search
plugin.validate() and _supports_capability(plugin, capability)
)
except Exception:
availability[info.canonical_name] = False
return availability
def list_search_plugin_names() -> List[str]:
"""Return registered search-provider names without instantiating plugins."""
def list_plugin_names_with_capability(capability: str) -> List[str]:
return sorted(
info.canonical_name
for info in REGISTRY.iter_providers()
if info.supports_search
)
def get_upload_plugin(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[FileProvider]:
plugin = get_plugin(name, config)
if plugin is None:
return None
if not _supports_upload(plugin):
debug(f"[plugin] Plugin '{name}' does not support upload")
return None
return plugin # type: ignore[return-value]
def list_upload_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
try:
plugin = info.provider_class(config)
availability[info.canonical_name] = bool(
plugin.validate() and info.supports_upload
)
except Exception:
availability[info.canonical_name] = False
return availability
def list_upload_plugin_names() -> List[str]:
"""Return registered upload-provider names without instantiating plugins."""
return sorted(
info.canonical_name
for info in REGISTRY.iter_providers()
if info.supports_upload
for info in REGISTRY.iter_plugins()
if _info_supports_capability(info, capability)
)
@@ -677,8 +674,8 @@ def match_plugin_name_for_url(url: str) -> Optional[str]:
return "openlibrary" if REGISTRY.has_name("openlibrary") else None
return "internetarchive" if REGISTRY.has_name("internetarchive") else None
for info in REGISTRY.iter_providers():
domains = _provider_url_patterns(info.provider_class)
for info in REGISTRY.iter_plugins():
domains = _plugin_url_patterns(info.plugin_class)
if not domains:
continue
for domain in domains:
@@ -721,12 +718,10 @@ def plugin_inline_query_choices(
mapping: Dict[str, List[Dict[str, Any]]] = {}
info = REGISTRY.get(pname)
if info is not None:
mapping = _collect_inline_choice_mapping(info.provider_class)
mapping = _collect_inline_choice_mapping(info.plugin_class)
if not mapping:
plugin = get_search_plugin(pname, config)
if plugin is None:
plugin = get_plugin(pname, config)
plugin = get_plugin(pname, config)
if plugin is None:
return []
mapping = _collect_inline_choice_mapping(plugin)
@@ -770,9 +765,9 @@ def get_plugin_for_url(url: str,
def list_selection_url_prefixes() -> List[str]:
prefixes: List[str] = []
seen: set[str] = set()
for info in REGISTRY.iter_providers():
for info in REGISTRY.iter_plugins():
try:
values = info.provider_class.selection_url_prefixes()
values = info.plugin_class.selection_url_prefixes()
except Exception:
values = ()
for value in values or ():
@@ -842,21 +837,17 @@ def resolve_inline_filters(
__all__ = [
"ProviderInfo",
"PluginInfo",
"Provider",
"SearchProvider",
"FileProvider",
"SearchResult",
"PluginRegistry",
"PLUGIN_REGISTRY",
"register_plugin",
"get_plugin",
"list_plugins",
"get_search_plugin",
"list_search_plugins",
"get_upload_plugin",
"list_upload_plugins",
"get_plugin_with_capability",
"list_plugins_with_capability",
"list_plugin_names_with_capability",
"match_plugin_name_for_url",
"get_plugin_for_url",
"list_selection_url_prefixes",
SYS/detail_view_helpers.py
+81 -3
View File
@@ -3,8 +3,41 @@ from __future__ import annotations
from typing import Any, Iterable, Optional, Sequence
_ACRONYM_LABELS = {
"id": "ID",
"ids": "IDs",
"url": "URL",
"urls": "URLs",
"api": "API",
"http": "HTTP",
"https": "HTTPS",
"ftp": "FTP",
"ftps": "FTPS",
"scp": "SCP",
"ssh": "SSH",
"ip": "IP",
"mpv": "MPV",
}
def _labelize_key(key: str) -> str:
return str(key or "").replace("_", " ").title()
parts = [part for part in str(key or "").replace("_", " ").split() if part]
normalized: list[str] = []
for part in parts:
lowered = part.lower()
normalized.append(_ACRONYM_LABELS.get(lowered, part.title()))
return " ".join(normalized)
def _has_display_value(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
text = value.strip()
return bool(text and text.lower() not in {"<null>", "null", "none"})
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return any(_has_display_value(item) for item in value)
return True
def _normalize_tags_value(tags: Any) -> Optional[str]:
@@ -45,7 +78,7 @@ def prepare_detail_metadata(
if str(key).startswith("_") or key in {"selection_action", "selection_args"}:
continue
label = _labelize_key(str(key))
if label not in metadata and value is not None:
if label not in metadata and _has_display_value(value):
metadata[label] = value
if title:
@@ -62,7 +95,7 @@ def prepare_detail_metadata(
metadata["Tags"] = tags_text
for key, value in (extra_fields or {}).items():
if value is not None:
if _has_display_value(value):
metadata[str(key)] = value
return metadata
@@ -77,6 +110,7 @@ def create_detail_view(
init_command: Optional[tuple[str, Sequence[str]]] = None,
max_columns: Optional[int] = None,
exclude_tags: bool = False,
detail_order: Optional[Sequence[str]] = None,
value_case: Optional[str] = "preserve",
perseverance: bool = True,
) -> Any:
@@ -87,6 +121,8 @@ def create_detail_view(
kwargs["max_columns"] = max_columns
if exclude_tags:
kwargs["exclude_tags"] = True
if detail_order is not None:
kwargs["detail_order"] = list(detail_order)
table = ItemDetailView(title, **kwargs)
if table_name:
@@ -102,3 +138,45 @@ def create_detail_view(
name, args = init_command
table = table.init_command(name, list(args))
return table
def render_selection_detail_view(
*,
ctx: Any,
item: Any,
title: str,
metadata: dict[str, Any],
table_name: Optional[str] = None,
detail_order: Optional[Sequence[str]] = None,
value_case: Optional[str] = "preserve",
exclude_tags: bool = False,
) -> bool:
from SYS.rich_display import stdout_console
detail_view = create_detail_view(
title,
metadata,
table_name=table_name,
detail_order=detail_order,
value_case=value_case,
exclude_tags=exclude_tags,
)
payload = item.to_dict() if hasattr(item, "to_dict") else item
try:
if hasattr(ctx, "set_last_result_items_only") and isinstance(payload, dict):
ctx.set_last_result_items_only([payload])
except Exception:
pass
try:
try:
detail_view.title = ""
detail_view.header_lines = []
except Exception:
pass
stdout_console().print()
stdout_console().print(detail_view.to_rich())
except Exception:
return False
return True
SYS/pipeline.py
+37 -7
View File
@@ -1478,13 +1478,18 @@ class PipelineExecutor:
config: Any,
selected_items: list,
*,
stage_is_last: bool
stage_is_last: bool,
source_command: Any = None,
prefer_detail_fallback: bool = False,
) -> bool:
if not stage_is_last:
return False
candidates: list[str] = []
seen: set[str] = set()
current_table = None
table_meta = None
table_type = ""
def _add(value) -> None:
try:
@@ -1504,6 +1509,8 @@ class PipelineExecutor:
table if current_table and hasattr(current_table,
"table") else None
)
if current_table and hasattr(current_table, "table"):
table_type = str(getattr(current_table, "table", "") or "").strip()
# Prefer an explicit plugin hint from table metadata when available.
# This keeps @N selectors working even when row payloads don't carry a
@@ -1516,6 +1523,7 @@ class PipelineExecutor:
)
except Exception:
meta = None
table_meta = meta if isinstance(meta, dict) else None
if isinstance(meta, dict):
_add(meta.get("plugin"))
_add(meta.get("provider"))
@@ -1585,6 +1593,26 @@ class PipelineExecutor:
if handled:
return True
if prefer_detail_fallback:
detail_renderer = getattr(provider, "show_selection_details", None)
if callable(detail_renderer):
try:
detail_handled = bool(
detail_renderer(
selected_items,
ctx=ctx,
stage_is_last=True,
source_command=str(source_command or ""),
table_type=table_type,
table_metadata=table_meta,
)
)
except Exception as exc:
logger.exception("%s detail fallback failed during selection: %s", key, exc)
return True
if detail_handled:
return True
@staticmethod
def _maybe_expand_plugin_selection(
selected_items: List[Any],
@@ -2180,10 +2208,12 @@ class PipelineExecutor:
filtered = expanded
if PipelineExecutor._maybe_run_class_selector(
ctx,
config,
filtered,
stage_is_last=(not stages)):
ctx,
config,
filtered,
stage_is_last=(not stages),
source_command=source_cmd,
prefer_detail_fallback=bool(prefer_row_action and not stages and len(selection_indices) == 1)):
return False, None
from SYS.pipe_object import coerce_to_pipe_object
@@ -2204,7 +2234,7 @@ class PipelineExecutor:
except Exception:
logger.exception("Failed to record Applied @N selection log step (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None))
# Auto-insert downloader stages for provider tables.
# Auto-insert downloader stages for plugin tables.
try:
current_table = ctx.get_current_stage_table()
if current_table is None and hasattr(ctx, "get_display_table"):
@@ -2360,7 +2390,7 @@ class PipelineExecutor:
# Multi-selection fallback: if any selected row declares a
# download-file action, insert a generic download-file stage.
# This keeps provider-specific behavior in provider metadata.
# This keeps plugin-specific behavior in plugin metadata.
if (not inserted_provider_download) and len(selection_indices) > 1:
try:
has_download_row_action = False
SYS/plugin_config.py
-12
View File
@@ -68,10 +68,6 @@ def get_plugin_schema(plugin_name: str) -> List[ConfigField]:
return _call_schema(plugin_class, f"plugin '{plugin_name}'")
def get_provider_schema(provider_name: str) -> List[ConfigField]:
return get_plugin_schema(provider_name)
def get_tool_schema(tool_name: str) -> List[ConfigField]:
tool_name = str(tool_name or "").strip()
if not tool_name:
@@ -149,10 +145,6 @@ def build_default_plugin_config(plugin_name: str) -> Dict[str, Any]:
return config
def build_default_provider_config(provider_name: str) -> Dict[str, Any]:
return build_default_plugin_config(provider_name)
def build_default_tool_config(tool_name: str) -> Dict[str, Any]:
config: Dict[str, Any] = {}
for field in get_tool_schema(tool_name):
@@ -215,10 +207,6 @@ def get_configurable_plugin_types() -> List[str]:
return sorted(set(options))
def get_configurable_provider_types() -> List[str]:
return get_configurable_plugin_types()
def get_configurable_tool_types() -> List[str]:
options: List[str] = []
try:
SYS/result_table.py
+59 -12
View File
@@ -724,7 +724,7 @@ class Table:
"""Table type (e.g., 'youtube', 'soulseek') for context-aware selection logic."""
self.table_metadata: Dict[str, Any] = {}
"""Optional provider/table metadata (e.g., provider name, view)."""
"""Optional plugin/table metadata (e.g., plugin name, view)."""
self.value_case: str = "preserve"
"""Display-only value casing: 'lower', 'upper', or 'preserve' (default)."""
@@ -754,12 +754,12 @@ class Table:
return self
def set_table_metadata(self, metadata: Optional[Dict[str, Any]]) -> "Table":
"""Attach provider/table metadata for downstream selection logic."""
"""Attach plugin/table metadata for downstream selection logic."""
self.table_metadata = dict(metadata or {})
return self
def get_table_metadata(self) -> Dict[str, Any]:
"""Return attached provider/table metadata (copy to avoid mutation)."""
"""Return attached plugin/table metadata (copy to avoid mutation)."""
try:
return dict(self.table_metadata)
except Exception:
@@ -2224,6 +2224,34 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
out = {}
def _merge_columns(columns_value: Any) -> None:
if not isinstance(columns_value, (list, tuple)):
return
for column in columns_value:
label = None
value = None
if isinstance(column, (list, tuple)) and len(column) >= 2:
label, value = column[0], column[1]
elif isinstance(column, dict):
label = column.get("name") or column.get("label") or column.get("key")
value = column.get("value")
else:
label = getattr(column, "name", None)
value = getattr(column, "value", None)
label_text = str(label or "").strip()
if not label_text or value is None:
continue
value_text = str(value).strip()
if not value_text:
continue
normalized = label_text.lower()
if any(str(existing or "").strip().lower() == normalized for existing in out):
continue
out[label_text] = value_text
# Handle ResultModel specifically for better detail display
if ResultModel is not None and isinstance(item, ResultModel):
if item.title: out["Title"] = item.title
@@ -2231,6 +2259,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
if item.ext: out["Ext"] = item.ext
if item.size_bytes: out["Size"] = format_mb(item.size_bytes)
if item.source: out["Store"] = item.source
_merge_columns(getattr(item, "columns", None))
# Merge internal metadata dict
if item.metadata:
@@ -2256,6 +2285,7 @@ def extract_item_metadata(item: Any) -> Dict[str, Any]:
# Fallback to existing extraction logic for legacy objects/dicts
# Convert once and reuse throughout to avoid repeated _as_dict() calls
data = _as_dict(item) or {}
_merge_columns(data.get("columns"))
# Use existing extractors from match-standard result table columns
title = extract_title_value(item)
@@ -2350,12 +2380,14 @@ class ItemDetailView(Table):
item_metadata: Optional[Dict[str, Any]] = None,
detail_title: Optional[str] = None,
exclude_tags: bool = False,
detail_order: Optional[List[str]] = None,
**kwargs
):
super().__init__(title, **kwargs)
self.item_metadata = item_metadata or {}
self.detail_title = detail_title
self.exclude_tags = exclude_tags
self.detail_order = [str(value) for value in (detail_order or []) if str(value or "").strip()]
def to_rich(self):
"""Render the item details panel above the standard results table."""
@@ -2406,8 +2438,26 @@ class ItemDetailView(Table):
return Group(*renderables)
# Canonical display order for metadata
order = ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"]
def _has_renderable_value(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
text = value.strip()
return bool(text and text.lower() not in {"<null>", "null", "none"})
if isinstance(value, (list, tuple, set)):
return any(_has_renderable_value(item) for item in value)
return True
# Canonical display order for metadata; plugin-specific detail views can
# prepend a preferred order without needing to reimplement rendering.
order: List[str] = []
seen_order: set[str] = set()
for key in list(self.detail_order or []) + ["Title", "Hash", "Store", "Path", "Ext", "Size", "Duration", "Url", "Relations"]:
normalized = str(key or "").strip().lower()
if not normalized or normalized in seen_order:
continue
seen_order.add(normalized)
order.append(str(key))
has_details = False
# Add ordered items first
@@ -2431,19 +2481,16 @@ class ItemDetailView(Table):
else:
val = "\n".join([f"[dim]→[/dim] {r}" for r in val])
if val is not None and val != "":
if _has_renderable_value(val):
details_table.add_row(f"{key}:", str(val))
has_details = True
elif key in ["Url", "Relations", "Ext"]:
# Show <null> for these important identifier fields if blank
details_table.add_row(f"{key}:", "[dim]<null>[/dim]")
has_details = True
# Add any remaining metadata not in the canonical list
ordered_keys = {x.lower() for x in order}
for k, v in self.item_metadata.items():
k_norm = k.lower()
if k_norm not in [x.lower() for x in order] and v and k_norm not in ["tags", "tag"]:
label = k.capitalize() if len(k) > 1 else k.upper()
if k_norm not in ordered_keys and _has_renderable_value(v) and k_norm not in ["tags", "tag"]:
label = str(k or "")
details_table.add_row(f"{label}:", str(v))
has_details = True
SYS/rich_display.py
+12 -12
View File
@@ -77,26 +77,26 @@ def capture_rich_output(*, stdout: TextIO, stderr: TextIO) -> Iterator[None]:
_STDERR_CONSOLE = previous_stderr
def show_provider_config_panel(
provider_names: str | List[str],
def show_plugin_config_panel(
plugin_names: str | List[str],
) -> None:
"""Show a Rich panel explaining how to configure providers."""
"""Show a Rich panel explaining how to configure plugins."""
from rich.table import Table as RichTable
from rich.console import Group
if isinstance(provider_names, str):
providers = [p.strip() for p in provider_names.split(",")]
if isinstance(plugin_names, str):
plugins = [p.strip() for p in plugin_names.split(",")]
else:
providers = provider_names
plugins = plugin_names
table = RichTable.grid(padding=(0, 1))
table.add_column(style="bold red")
for provider in providers:
table.add_row(f" • {provider}")
for plugin in plugins:
table.add_row(f" • {plugin}")
group = Group(
Text("The following providers are not configured and cannot be used:\n"),
Text("The following plugins are not configured and cannot be used:\n"),
table,
Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.")
)
@@ -147,17 +147,17 @@ def show_store_config_panel(
stdout_console().print(panel)
def show_available_providers_panel(provider_names: List[str]) -> None:
def show_available_plugins_panel(plugin_names: List[str]) -> None:
"""Show a Rich panel listing available/configured plugins."""
from rich.columns import Columns
from rich.console import Group
if not provider_names:
if not plugin_names:
return
# Use Columns to display them efficiently in the panel
cols = Columns(
[f"[bold green] \u2713 [/bold green]{p}" for p in sorted(provider_names)],
[f"[bold green] \u2713 [/bold green]{p}" for p in sorted(plugin_names)],
equal=True,
column_first=True,
expand=True
Store/HydrusNetwork.py
+18 -17
View File
@@ -4,6 +4,7 @@ import re
import sys
import tempfile
import shutil
from collections.abc import Mapping, Sequence as SequenceABC
from collections import deque
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
@@ -169,7 +170,7 @@ class HydrusNetwork(Store):
api_key: Hydrus Client API access key
url: Hydrus client URL (e.g., 'http://192.168.1.230:45869')
"""
from API.HydrusNetwork import HydrusNetwork as HydrusClient
from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient
if instance_name is None and NAME is not None:
instance_name = str(NAME)
@@ -713,7 +714,7 @@ class HydrusNetwork(Store):
"""Best-effort URL search by scanning Hydrus metadata with include_file_url=True."""
try:
from API.HydrusNetwork import _generate_hydrus_url_variants
from plugins.hydrusnetwork.api import _generate_hydrus_url_variants
except Exception:
_generate_hydrus_url_variants = None # type: ignore[assignment]
@@ -808,7 +809,7 @@ class HydrusNetwork(Store):
self._has_url_predicate = True
except Exception as exc:
try:
from API.HydrusNetwork import HydrusRequestError
from plugins.hydrusnetwork.api import HydrusRequestError
if isinstance(exc, HydrusRequestError) and getattr(exc, "status", None) == 400:
self._has_url_predicate = False
@@ -1844,7 +1845,7 @@ class HydrusNetwork(Store):
extracted_tags = self._extract_tags_from_hydrus_meta(
meta,
service_key=None,
service_name="my tags",
service_name=None,
)
for raw_tag in extracted_tags:
tag_text = str(raw_tag or "").strip()
@@ -2201,7 +2202,7 @@ class HydrusNetwork(Store):
return []
try:
from API.HydrusNetwork import HydrusRequestSpec, _generate_hydrus_url_variants
from plugins.hydrusnetwork.api import HydrusRequestSpec, _generate_hydrus_url_variants
except Exception:
return []
@@ -2339,7 +2340,7 @@ class HydrusNetwork(Store):
try:
return client.get_url_info(u) # type: ignore[attr-defined]
except Exception:
from API.HydrusNetwork import HydrusRequestSpec
from plugins.hydrusnetwork.api import HydrusRequestSpec
spec = HydrusRequestSpec(
method="GET",
@@ -2563,7 +2564,7 @@ class HydrusNetwork(Store):
meta: Dict[str,
Any],
service_key: Optional[str],
service_name: str
service_name: Optional[str]
) -> List[str]:
"""Extract current tags from Hydrus metadata dict.
@@ -2571,7 +2572,7 @@ class HydrusNetwork(Store):
Falls back to storage_tags status '0' (current).
"""
tags_payload = meta.get("tags")
if not isinstance(tags_payload, dict):
if not isinstance(tags_payload, Mapping):
return []
desired_service_name = str(service_name or "").strip().lower()
@@ -2593,20 +2594,20 @@ class HydrusNetwork(Store):
out.append(cleaned)
def _collect_current(container: Any, out: List[str]) -> None:
if isinstance(container, list):
if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)):
for tag in container:
_append_tag(out, tag)
return
if isinstance(container, dict):
if isinstance(container, Mapping):
current = container.get("0")
if current is None:
current = container.get(0)
if isinstance(current, list):
if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)):
for tag in current:
_append_tag(out, tag)
def _collect_service_data(service_data: Any, out: List[str]) -> None:
if not isinstance(service_data, dict):
if not isinstance(service_data, Mapping):
return
display = (
@@ -2630,7 +2631,7 @@ class HydrusNetwork(Store):
if not collected and desired_service_name:
for maybe_service in tags_payload.values():
if not isinstance(maybe_service, dict):
if not isinstance(maybe_service, Mapping):
continue
svc_name = str(
maybe_service.get("service_name")
@@ -2642,11 +2643,11 @@ class HydrusNetwork(Store):
names_map = tags_payload.get("service_keys_to_names")
statuses_map = tags_payload.get("service_keys_to_statuses_to_tags")
if isinstance(statuses_map, dict):
if isinstance(statuses_map, Mapping):
keys_to_collect: List[str] = []
if desired_service_key:
keys_to_collect.append(desired_service_key)
if desired_service_name and isinstance(names_map, dict):
if desired_service_name and isinstance(names_map, Mapping):
for raw_key, raw_name in names_map.items():
if str(raw_name or "").strip().lower() == desired_service_name:
keys_to_collect.append(str(raw_key))
@@ -2663,7 +2664,7 @@ class HydrusNetwork(Store):
_collect_service_data(maybe_service, collected)
top_level_tags = meta.get("tags_flat")
if isinstance(top_level_tags, list):
if isinstance(top_level_tags, SequenceABC) and not isinstance(top_level_tags, (str, bytes, bytearray, Mapping)):
_collect_current(top_level_tags, collected)
deduped: List[str] = []
@@ -2682,7 +2683,7 @@ class HydrusNetwork(Store):
tags = HydrusNetwork._extract_tags_from_hydrus_meta(
meta,
service_key=None,
service_name="my tags",
service_name=None,
)
normalized_tags: List[str] = []
TUI/modalscreen/config_modal.py
+28 -28
View File
@@ -22,10 +22,10 @@ from SYS.config import (
from SYS.database import db
from SYS.logger import log, debug
from SYS.plugin_config import (
build_default_provider_config,
build_default_plugin_config,
build_default_store_config,
build_default_tool_config,
get_configurable_provider_types,
get_configurable_plugin_types,
get_configurable_store_types,
get_configurable_tool_types,
get_global_schema,
@@ -161,7 +161,7 @@ class ConfigModal(ModalScreen):
# Load config from the workspace root (parent of SYS)
self.config_data = load_config()
self.current_category = "globals"
self.editing_item_type = None # 'store' or 'provider'
self.editing_item_type = None # 'store' or 'plugin'
self.editing_item_name = None
self._button_id_map = {}
self._provider_button_map: Dict[str, tuple[str, str]] = {}
@@ -598,14 +598,14 @@ class ConfigModal(ModalScreen):
row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn"))
idx += 1
if item_type == "provider" and isinstance(item_name, str):
provider = self._instantiate_provider_for_editor(item_name, self.config_data)
if item_type == "plugin" and isinstance(item_name, str):
provider = self._instantiate_plugin_for_editor(item_name, self.config_data)
if provider is not None:
provider_actions = provider.config_actions() or []
if provider_actions:
container.mount(Rule())
container.mount(Label(f"{provider.label} helpers", classes="config-label"))
helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip()
helper_text = str(provider.config_helper_text() or "Use these helpers to validate plugin settings.").strip()
status = Static(helper_text, id="provider-status")
container.mount(status)
self._provider_status = status
@@ -626,7 +626,7 @@ class ConfigModal(ModalScreen):
)
if (
item_type == "provider"
item_type == "plugin"
and isinstance(item_name, str)
and item_name.strip().lower() == "matrix"
):
@@ -870,12 +870,12 @@ class ConfigModal(ModalScreen):
self.refresh_view()
elif bid in self._provider_button_map:
provider_name, action_id = self._provider_button_map[bid]
self._request_provider_action(provider_name, action_id)
self._request_plugin_action(provider_name, action_id)
elif bid == "add-store-btn":
options = get_configurable_store_types()
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
elif bid == "add-provider-btn":
options = get_configurable_provider_types()
options = get_configurable_plugin_types()
self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected)
elif bid == "add-tool-btn":
options = get_configurable_tool_types() or ["ytdlp"]
@@ -971,46 +971,46 @@ class ConfigModal(ModalScreen):
else:
self.notify("Clipboard not supported in this terminal", severity="warning")
def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]:
def _instantiate_plugin_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]:
try:
provider_class = get_plugin_class(provider_name)
plugin_class = get_plugin_class(provider_name)
except Exception:
provider_class = None
if provider_class is None:
plugin_class = None
if plugin_class is None:
return None
try:
return provider_class(config_data or self.config_data)
return plugin_class(config_data or self.config_data)
except Exception:
logger.exception("Failed to instantiate provider '%s' for config helper", provider_name)
logger.exception("Failed to instantiate plugin '%s' for config helper", provider_name)
return None
def _request_provider_action(self, provider_name: str, action_id: str) -> None:
def _request_plugin_action(self, provider_name: str, action_id: str) -> None:
if self._provider_action_running:
return
self._synchronize_inputs_to_config()
self._provider_action_running = True
if self._provider_status is not None:
self._provider_status.update(f"Running {action_id.replace('_', ' ')}…")
self._provider_action_background(provider_name, action_id, deepcopy(self.config_data))
self._plugin_action_background(provider_name, action_id, deepcopy(self.config_data))
@work(thread=True)
def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None:
def _plugin_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None:
try:
provider = self._instantiate_provider_for_editor(provider_name, config_snapshot)
provider = self._instantiate_plugin_for_editor(provider_name, config_snapshot)
if provider is None:
raise RuntimeError(f"Provider '{provider_name}' is unavailable")
raise RuntimeError(f"Plugin '{provider_name}' is unavailable")
result = provider.run_config_action(action_id)
if not isinstance(result, dict):
result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."}
result = {"ok": False, "message": f"Plugin '{provider_name}' returned an invalid config action result."}
except Exception as exc:
result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."}
result = {"ok": False, "message": str(exc) or f"Plugin action '{action_id}' failed."}
try:
self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result)
self.app.call_from_thread(self._plugin_action_complete, provider_name, action_id, result)
except Exception:
self._provider_action_complete(provider_name, action_id, result)
self._plugin_action_complete(provider_name, action_id, result)
def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None:
def _plugin_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None:
self._provider_action_running = False
ok = bool(result.get("ok"))
message = str(result.get("message") or f"Provider action '{action_id}' finished.")
@@ -1075,11 +1075,11 @@ class ConfigModal(ModalScreen):
if "provider" not in self.config_data:
self.config_data["provider"] = {}
# For providers, they are usually top-level entries in 'provider' dict
# Plugins are configured under the top-level 'provider' dict for now.
if ptype not in self.config_data["provider"]:
self.config_data["provider"][ptype] = build_default_provider_config(ptype)
self.config_data["provider"][ptype] = build_default_plugin_config(ptype)
self.editing_item_type = "provider"
self.editing_item_type = "plugin"
self.editing_item_name = ptype
self.refresh_view()
TUI/modalscreen/search.py
+3 -3
View File
@@ -16,7 +16,7 @@ import asyncio
sys.path.insert(0, str(Path(__file__).parent.parent))
from SYS.config import load_config, resolve_output_dir
from SYS.result_table import Table
from ProviderCore.registry import get_search_plugin
from ProviderCore.registry import get_plugin_with_capability
logger = logging.getLogger(__name__)
@@ -174,7 +174,7 @@ class SearchModal(ModalScreen):
self.current_worker.log_step(f"Connecting to {source}...")
try:
provider = get_search_plugin(source)
provider = get_plugin_with_capability(source, "search")
if not provider:
logger.error(f"[search-modal] Provider not available: {source}")
if self.current_worker:
@@ -380,7 +380,7 @@ class SearchModal(ModalScreen):
config = load_config()
output_dir = resolve_output_dir(config)
provider = get_search_plugin("openlibrary", config=config)
provider = get_plugin_with_capability("openlibrary", "search", config=config)
if not provider:
logger.error("[search-modal] Provider not available: openlibrary")
return
cmdlet/_shared.py
+10 -2
View File
@@ -190,10 +190,18 @@ class SharedArgs:
name="store",
type="enum",
choices=[], # Dynamically populated via get_store_choices()
description="Selects store",
description="Selects a storage backend",
query_key="store",
)
INSTANCE = CmdletArg(
name="instance",
type="string",
description="Selects a plugin instance",
query_key="instance",
query_aliases=["store"],
)
URL = CmdletArg(
name="url",
type="string",
@@ -1410,7 +1418,7 @@ def fetch_hydrus_metadata(
Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
Args:
config: Configuration object used to resolve the Hydrus provider/store
config: Configuration object used to resolve the Hydrus plugin/store
hash_hex: File hash to fetch metadata for
store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
cmdlet/add_file.py
+42 -9
View File
@@ -15,6 +15,7 @@ from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store
from API.HTTP import _download_direct_file
@@ -178,10 +179,11 @@ class Add_File(Cmdlet):
summary=
"Ingest a local media file to a store backend, upload plugin, or local directory.",
usage=
"add-file (-path <filepath> | <piped>) (-storage <location> | -plugin <upload-plugin>) [-delete]",
"add-file (-path <filepath> | <piped>) (-store <backend|path> | -plugin <upload-plugin>) [-instance NAME] [-delete]",
arg=[
SharedArgs.PATH,
SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.URL,
SharedArgs.PLUGIN,
CmdletArg(
@@ -194,7 +196,7 @@ class Add_File(Cmdlet):
],
detail=[
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
"- Storage location options (use -storage):",
"- Storage location options (use -store):",
" hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory",
" <path>: Copy file to specified directory",
@@ -202,9 +204,12 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- In plugin mode, -store <name> is still accepted as a compatibility alias for -instance <name>.",
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
],
exec=self.run,
)
@@ -223,9 +228,12 @@ class Add_File(Cmdlet):
path_arg = parsed.get("path")
location = parsed.get("store")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url")
plugin_name = parsed.get("plugin")
delete_after = parsed.get("delete", False)
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Convenience: when piping a file into add-file, allow `-path <existing dir>`
# to act as the destination export directory.
@@ -412,6 +420,7 @@ class Add_File(Cmdlet):
("items", total_items),
("location", location),
("plugin", plugin_name),
("instance", plugin_instance),
("delete", delete_after),
],
border_style="cyan",
@@ -647,6 +656,7 @@ class Add_File(Cmdlet):
code = self._handle_plugin_upload(
media_path,
plugin_name,
plugin_instance,
pipe_obj,
config,
delete_after_item
@@ -1442,9 +1452,9 @@ class Add_File(Cmdlet):
if not plugin_key:
return None, None, None
from ProviderCore.registry import get_search_plugin
from ProviderCore.registry import get_plugin
plugin = get_search_plugin(plugin_key, config)
plugin = get_plugin(plugin_key, config)
if plugin is None:
return None, None, None
@@ -1762,6 +1772,7 @@ class Add_File(Cmdlet):
*,
hash_value: str,
store: str,
provider: Optional[str] = None,
path: Optional[str],
tag: List[str],
title: Optional[str],
@@ -1770,6 +1781,7 @@ class Add_File(Cmdlet):
) -> None:
pipe_obj.hash = hash_value
pipe_obj.store = store
pipe_obj.provider = provider
pipe_obj.is_temp = False
pipe_obj.path = path
pipe_obj.tag = tag
@@ -2180,23 +2192,42 @@ class Add_File(Cmdlet):
def _handle_plugin_upload(
media_path: Path,
plugin_name: str,
instance_name: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
delete_after: bool,
) -> int:
"""Handle uploading via an upload plugin (e.g. 0x0)."""
from ProviderCore.registry import get_upload_plugin
from ProviderCore.registry import (
get_plugin_with_capability,
list_plugin_names_with_capability,
list_plugins_with_capability,
)
log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr)
try:
file_provider = get_upload_plugin(plugin_name, config)
file_provider = get_plugin_with_capability(plugin_name, "upload", config)
if not file_provider:
log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr)
available_map = list_plugins_with_capability("upload", config)
known_upload_plugins = set(list_plugin_names_with_capability("upload"))
available_uploads = [name for name, enabled in available_map.items() if enabled and name in known_upload_plugins]
if str(plugin_name or "").strip().lower() in known_upload_plugins:
show_plugin_config_panel([plugin_name])
else:
log(f"Upload plugin '{plugin_name}' is not available or does not support upload", file=sys.stderr)
if available_uploads:
show_available_plugins_panel(sorted(available_uploads))
return 1
hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj)
hoster_url = file_provider.upload(
str(media_path),
pipe_obj=pipe_obj,
instance=instance_name,
)
log(f"File uploaded: {hoster_url}", file=sys.stderr)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
@@ -2209,6 +2240,7 @@ class Add_File(Cmdlet):
extra_updates: Dict[str,
Any] = {
"plugin": plugin_name,
"instance": instance_name,
"plugin_url": hoster_url,
}
if isinstance(pipe_obj.extra, dict):
@@ -2222,7 +2254,8 @@ class Add_File(Cmdlet):
Add_File._update_pipe_object_destination(
pipe_obj,
hash_value=f_hash or "unknown",
store=plugin_name or "plugin",
store="",
provider=plugin_name or None,
path=file_path,
tag=pipe_obj.tag,
title=pipe_obj.title or (media_path.name if media_path else None),
cmdlet/download_file.py
+13 -13
View File
@@ -55,12 +55,13 @@ class Download_File(Cmdlet):
name="download-file",
summary="Download files or streaming media",
usage=
"download-file <url> [-path DIR] [options] OR @N | download-file [-path DIR|DIR] [options]",
"download-file <url> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]",
alias=["dl-file",
"download-http"],
arg=[
SharedArgs.URL,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
SharedArgs.QUERY,
QueryArg(
@@ -85,6 +86,7 @@ class Download_File(Cmdlet):
],
detail=[
"Download files directly via HTTP or streaming media via yt-dlp.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
],
exec=self.run,
@@ -522,13 +524,13 @@ class Download_File(Cmdlet):
config: Dict[str,
Any],
) -> List[Any]:
get_search_plugin = registry.get("get_search_plugin")
get_provider = registry.get("get_plugin")
expanded_items: List[Any] = []
for item in piped_items:
try:
provider_key = self._provider_key_from_item(item)
provider = get_search_plugin(provider_key, config) if provider_key and get_search_plugin else None
provider = get_provider(provider_key, config) if provider_key and get_provider else None
# Generic hook: If provider has expand_item(item), use it.
if provider and hasattr(provider, "expand_item") and callable(provider.expand_item):
@@ -566,7 +568,7 @@ class Download_File(Cmdlet):
) -> tuple[int, int]:
downloaded_count = 0
queued_magnet_submissions = 0
get_search_plugin = registry.get("get_search_plugin")
get_provider = registry.get("get_plugin")
SearchResult = registry.get("SearchResult")
expanded_items = self._expand_provider_items(
@@ -622,15 +624,15 @@ class Download_File(Cmdlet):
transfer_label = label
# If this looks like a provider item and providers are available, prefer provider.download()
# If this looks like a plugin-owned item and a plugin is available, prefer plugin.download().
downloaded_path: Optional[Path] = None
attempted_provider_download = False
provider_sr = None
provider_obj = None
provider_key = self._provider_key_from_item(item)
if provider_key and get_search_plugin and SearchResult:
# Reuse helper to derive the provider key from table/provider/source hints.
provider_obj = get_search_plugin(provider_key, config)
if provider_key and get_provider and SearchResult:
# Reuse helper to derive the plugin key from table/plugin/source hints.
provider_obj = get_provider(provider_key, config)
if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False):
try:
@@ -697,7 +699,7 @@ class Download_File(Cmdlet):
)
continue
# Allow providers to add/enrich tags and metadata during download.
# Allow plugins to add or enrich tags and metadata during download.
if provider_sr is not None:
try:
sr_md = getattr(provider_sr, "full_metadata", None)
@@ -838,9 +840,9 @@ class Download_File(Cmdlet):
notes: Optional[Dict[str, str]] = None
try:
if isinstance(full_metadata, dict):
# Providers attach pre-built notes under the generic "_notes" key
# Plugins attach pre-built notes under the generic "_notes" key
# (e.g. Tidal sets {"lyric": subtitles} during download enrichment).
# This keeps provider-specific metadata handling inside the provider.
# This keeps plugin-specific metadata handling inside the plugin.
_provider_notes = full_metadata.get("_notes")
if isinstance(_provider_notes, dict) and _provider_notes:
notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v}
@@ -949,7 +951,6 @@ class Download_File(Cmdlet):
return {
"get_plugin": getattr(provider_registry, "get_plugin", None),
"get_search_plugin": getattr(provider_registry, "get_search_plugin", None),
"match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
"list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
"SearchResult": SearchResult,
@@ -957,7 +958,6 @@ class Download_File(Cmdlet):
except Exception:
return {
"get_plugin": None,
"get_search_plugin": None,
"match_plugin_name_for_url": None,
"list_selection_url_prefixes": None,
"SearchResult": None,
cmdlet/get_tag.py
+22 -22
View File
@@ -15,10 +15,10 @@ import sys
from SYS.logger import log, debug
from plugins.metadata_provider import (
get_default_subject_scrape_provider,
get_metadata_provider,
get_metadata_provider_for_url,
list_metadata_providers,
get_default_subject_scrape_plugin,
get_metadata_plugin,
get_metadata_plugin_for_url,
list_metadata_plugins,
scrape_isbn_metadata,
scrape_openlibrary_metadata,
)
@@ -393,33 +393,33 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
scrape_url = parsed_args.get("scrape")
scrape_requested = scrape_flag_present or scrape_url is not None
# Handle URL or provider scraping mode.
# Handle URL or metadata-plugin scraping mode.
if scrape_requested:
import json as json_module
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
provider = None
plugin = None
if scrape_target.startswith(("http://", "https://")):
provider = get_metadata_provider_for_url(scrape_target, config)
if provider is None:
log("No metadata provider can scrape this URL", file=sys.stderr)
plugin = get_metadata_plugin_for_url(scrape_target, config)
if plugin is None:
log("No metadata plugin can scrape this URL", file=sys.stderr)
return 1
payload = provider.scrape_url_payload(scrape_target)
payload = plugin.scrape_url_payload(scrape_target)
if not isinstance(payload, dict):
log(f"No metadata extracted from URL via {provider.name}", file=sys.stderr)
log(f"No metadata extracted from URL via {plugin.name}", file=sys.stderr)
return 1
print(json_module.dumps(payload, ensure_ascii=False))
return 0
if scrape_target:
provider = get_metadata_provider(scrape_target, config)
plugin = get_metadata_plugin(scrape_target, config)
else:
provider = get_default_subject_scrape_provider(config)
if provider is None:
plugin = get_default_subject_scrape_plugin(config)
if plugin is None:
if scrape_target:
log(f"Unknown metadata provider: {scrape_target}", file=sys.stderr)
log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
else:
log("No default metadata provider is available for subject scraping", file=sys.stderr)
log("No default metadata plugin is available for subject scraping", file=sys.stderr)
return 1
backend = None
@@ -548,7 +548,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
query_hint = resolved_subject_query or identifier_query or combined_query or title_hint
if not query_hint:
log(
f"No query could be resolved for metadata provider '{provider.name}'",
f"No query could be resolved for metadata plugin '{provider.name}'",
file=sys.stderr
)
return 1
@@ -749,9 +749,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
)
return 0
provider_for_apply = get_metadata_provider(str(result_provider), config)
if provider_for_apply is not None:
apply_tags = provider_for_apply.filter_tags_for_store_apply(
plugin_for_apply = get_metadata_plugin(str(result_provider), config)
if plugin_for_apply is not None:
apply_tags = plugin_for_apply.filter_tags_for_store_apply(
[str(t) for t in result_tags if t is not None]
)
else:
@@ -946,7 +946,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
_SCRAPE_CHOICES = []
try:
_SCRAPE_CHOICES = sorted(list_metadata_providers().keys())
_SCRAPE_CHOICES = sorted(list_metadata_plugins().keys())
except Exception:
_SCRAPE_CHOICES = [
"itunes",
@@ -1000,7 +1000,7 @@ class Get_Tag(Cmdlet):
' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")',
" -store: Store result to key for downstream pipeline",
" -emit: Quiet mode (no interactive selection)",
" -scrape: Scrape metadata from URL or metadata provider",
" -scrape: Scrape metadata from URL or metadata plugin",
],
exec=self.run,
)
cmdlet/search_file.py
+37 -17
View File
@@ -1,4 +1,4 @@
"""search-file cmdlet: Search for files in storage backends (Hydrus)."""
"""search-file cmdlet: Search store backends and search-capable plugins."""
from __future__ import annotations
@@ -15,11 +15,11 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import log, debug, debug_panel
from SYS.payload_builders import build_file_result_payload, normalize_file_extension
from ProviderCore.registry import get_search_plugin, list_search_plugins
from ProviderCore.registry import get_plugin_with_capability, list_plugins_with_capability
from SYS.rich_display import (
show_provider_config_panel,
show_plugin_config_panel,
show_store_config_panel,
show_available_providers_panel,
show_available_plugins_panel,
)
from SYS.database import insert_worker, update_worker, append_worker_stdout
from SYS.item_accessors import get_extension_field, get_int_field, get_result_title
@@ -164,13 +164,13 @@ def _summarize_worker_results(results: Sequence[Dict[str, Any]], preview_limit:
class search_file(Cmdlet):
"""Class-based search-file cmdlet for searching storage backends."""
"""Class-based search-file cmdlet for searching backends and providers."""
def __init__(self) -> None:
super().__init__(
name="search-file",
summary="Search storage backends (Hydrus) or external plugins (via -plugin).",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-plugin NAME]",
summary="Search configured store backends or search-capable plugins.",
usage="search-file [-query <query>] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]",
arg=[
CmdletArg(
"limit",
@@ -178,6 +178,7 @@ class search_file(Cmdlet):
description="Limit results (default: 100)"
),
SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.QUERY,
SharedArgs.PLUGIN,
CmdletArg(
@@ -187,8 +188,10 @@ class search_file(Cmdlet):
),
],
detail=[
"Search across storage backends: Hydrus instances",
"Use -store to search a specific backend by name",
"Search across configured store backends or plugin providers.",
"Use -store to target a specific store backend by name.",
"Use -plugin with -instance to target a named provider config.",
"In plugin mode, -store <name> is kept as a compatibility alias for -instance <name>.",
"URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png",
@@ -207,6 +210,7 @@ class search_file(Cmdlet):
"",
"Plugin search (-plugin):",
"search-file -plugin youtube 'tutorial' # Search YouTube plugin",
"search-file -plugin ftp -instance work '*' # Search a named FTP/SCP plugin instance",
"search-file -plugin alldebrid '*' # List AllDebrid magnets",
"search-file -plugin alldebrid -open 123 '*' # Show files for a magnet",
],
@@ -1451,6 +1455,7 @@ class search_file(Cmdlet):
self,
*,
plugin_name: str,
instance_name: Optional[str],
query: str,
limit: int,
limit_set: bool,
@@ -1475,15 +1480,15 @@ class search_file(Cmdlet):
log("Error: search-file -plugin requires both plugin and query", file=sys.stderr)
log(f"Usage: {self.usage}", file=sys.stderr)
providers_map = list_search_plugins(config)
providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a]
unconfigured = [n for n, a in providers_map.items() if not a]
if unconfigured:
show_provider_config_panel(unconfigured)
show_plugin_config_panel(unconfigured)
if available:
show_available_providers_panel(available)
show_available_plugins_panel(available)
return 1
@@ -1496,7 +1501,7 @@ class search_file(Cmdlet):
if hasattr(ctx_mod, "get_pipeline_state"):
progress = ctx_mod.get_pipeline_state().live_progress
provider = get_search_plugin(plugin_name, config)
provider = get_plugin_with_capability(plugin_name, "search", config)
if not provider:
if progress:
try:
@@ -1504,12 +1509,12 @@ class search_file(Cmdlet):
except Exception:
pass
show_provider_config_panel([plugin_name])
show_plugin_config_panel([plugin_name])
providers_map = list_search_plugins(config)
providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a]
if available:
show_available_providers_panel(available)
show_available_plugins_panel(available)
return 1
worker_id = str(uuid.uuid4())
@@ -1542,6 +1547,8 @@ class search_file(Cmdlet):
normalized_query = (normalized_query or "").strip()
query = normalized_query or "*"
search_filters = dict(provider_filters or {})
if instance_name and not search_filters.get("instance"):
search_filters["instance"] = str(instance_name).strip()
# Dynamic table generation via provider
table_title = provider.get_table_title(query, search_filters).strip().rstrip(":")
@@ -1564,6 +1571,7 @@ class search_file(Cmdlet):
"search-file provider request",
[
("provider", plugin_name),
("instance", search_filters.get("instance") or "<default>"),
("query", query),
("limit", limit),
("filters", search_filters or "<none>"),
@@ -1581,7 +1589,7 @@ class search_file(Cmdlet):
border_style="cyan",
)
# Allow providers to apply provider-specific UX transforms (e.g. auto-expansion)
# Allow plugins to apply plugin-specific UX transforms (e.g. auto-expansion)
try:
post = getattr(provider, "postprocess_search_results", None)
if callable(post) and isinstance(results, list):
@@ -1737,6 +1745,10 @@ class search_file(Cmdlet):
f.lower()
for f in (flag_registry.get("store") or {"-store", "--store"})
}
instance_flags = {
f.lower()
for f in (flag_registry.get("instance") or {"-instance", "--instance"})
}
limit_flags = {
f.lower()
for f in (flag_registry.get("limit") or {"-limit", "--limit"})
@@ -1753,6 +1765,7 @@ class search_file(Cmdlet):
# Parse arguments
query = ""
storage_backend: Optional[str] = None
instance_name: Optional[str] = None
plugin_name: Optional[str] = None
open_id: Optional[int] = None
limit = 100
@@ -1773,6 +1786,10 @@ class search_file(Cmdlet):
plugin_name = args_list[i + 1]
i += 2
continue
if low in instance_flags and i + 1 < len(args_list):
instance_name = args_list[i + 1]
i += 2
continue
if low in open_flags and i + 1 < len(args_list):
try:
open_id = int(args_list[i + 1])
@@ -1804,8 +1821,11 @@ class search_file(Cmdlet):
query = query.strip()
if plugin_name:
if storage_backend and not instance_name:
instance_name = storage_backend
return self._run_plugin_search(
plugin_name=plugin_name,
instance_name=instance_name,
query=query,
limit=limit,
limit_set=limit_set,
cmdnat/matrix.py
+1 -1
View File
@@ -493,7 +493,7 @@ def _maybe_download_hydrus_file(item: Any,
"""
try:
from SYS.config import get_hydrus_access_key, get_hydrus_url
from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file
from plugins.hydrusnetwork.api import HydrusNetwork as HydrusClient, download_hydrus_file
# Prefer per-item Hydrus instance name when it matches a configured instance.
store_name = None
cmdnat/pipe.py
+189 -117
View File
@@ -10,7 +10,7 @@ from datetime import datetime, timedelta
from urllib.parse import urlparse, parse_qs
from pathlib import Path
from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args
from ProviderCore.registry import get_plugin_for_url
from ProviderCore.registry import get_plugin, get_plugin_for_url
from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream
from SYS.result_table import Table
from MPV.mpv_ipc import MPV
@@ -32,6 +32,13 @@ _IPV4_RE = re.compile(r"^\d+\.\d+\.\d+\.\d+$")
_MPD_PATH_RE = re.compile(r"\.mpd($|\?)")
def _get_hydrus_provider(config: Optional[Dict[str, Any]] = None) -> Any:
try:
return get_plugin("hydrusnetwork", config or {})
except Exception:
return None
def _repo_root() -> Path:
try:
return Path(__file__).resolve().parent.parent
@@ -575,20 +582,51 @@ def _send_ipc_command(command: Dict[str, Any], silent: bool = False, wait: bool
return None
def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]:
def _extract_store_and_hash(
item: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> tuple[Optional[str], Optional[str]]:
store: Optional[str] = None
file_hash: Optional[str] = None
targets: list[str] = []
def _add_target(value: Any) -> None:
text = str(value or "").strip()
if text:
targets.append(text)
try:
if isinstance(item, dict):
store = item.get("store")
file_hash = item.get("hash") or item.get("file_hash")
_add_target(item.get("path"))
_add_target(item.get("url"))
_add_target(item.get("filename"))
metadata = item.get("full_metadata") or item.get("metadata")
else:
store = getattr(item, "store", None)
file_hash = getattr(item, "hash", None) or getattr(item, "file_hash", None)
_add_target(getattr(item, "path", None))
_add_target(getattr(item, "url", None))
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
except Exception:
store = None
file_hash = None
metadata = None
if isinstance(metadata, dict):
try:
if not store:
store = metadata.get("store")
if not file_hash or str(file_hash).strip().lower() in {"unknown", "none", "n/a", "na"}:
file_hash = metadata.get("hash") or metadata.get("hash_hex")
_add_target(metadata.get("path"))
_add_target(metadata.get("url"))
_add_target(metadata.get("selection_url"))
_add_target(metadata.get("hydrus_url"))
except Exception:
pass
try:
store = str(store).strip() if store else None
@@ -600,17 +638,37 @@ def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]:
except Exception:
file_hash = None
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is not None:
normalized_store = None
try:
if store and hydrus_provider.is_store_name(store):
normalized_store = store
except Exception:
normalized_store = None
for target in targets:
try:
parsed_store, parsed_hash = hydrus_provider.parse_hydrus_url(target)
except Exception:
parsed_store, parsed_hash = None, ""
if parsed_hash and not file_hash:
file_hash = parsed_hash
if parsed_store:
normalized_store = parsed_store
if normalized_store:
store = normalized_store
elif store and store.upper() in {"PATH", "LOCAL", "UNKNOWN"}:
store = None
if not file_hash:
try:
text = None
if isinstance(item, dict):
text = item.get("path") or item.get("url") or item.get("filename")
else:
text = getattr(item, "path", None) or getattr(item, "url", None)
if text:
for text in targets:
m = _SHA256_RE.search(str(text).lower())
if m:
file_hash = m.group(0)
break
except Exception:
pass
@@ -681,6 +739,8 @@ def _prefetch_notes_async(
set_notes_prefetch_pending(store, file_hash, True)
registry = Store(cfg, suppress_debug=True)
if not registry.is_available(str(store)):
return
backend = registry[str(store)]
notes = backend.get_note(str(file_hash), config=cfg) or {}
store_cached_notes(store, file_hash, notes)
@@ -718,7 +778,7 @@ def _schedule_notes_prefetch(items: Sequence[Any], config: Optional[Dict[str, An
seen: set[str] = set()
scheduled = 0
for item in items or []:
store, file_hash = _extract_store_and_hash(item)
store, file_hash = _extract_store_and_hash(item, config=config)
if not store or not file_hash:
continue
key = f"{store.lower()}:{file_hash}"
@@ -789,7 +849,12 @@ def _extract_target_from_memory_uri(text: str) -> Optional[str]:
return None
def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional[str]:
def _find_hydrus_instance_for_hash(
hash_str: str,
file_storage: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Find which Hydrus instance serves a specific file hash.
Args:
@@ -799,27 +864,29 @@ def _find_hydrus_instance_for_hash(hash_str: str, file_storage: Any) -> Optional
Returns:
Instance name (e.g., 'home') or None if not found
"""
# Query each Hydrus backend to see if it has this file
for backend_name in file_storage.list_backends():
backend = file_storage[backend_name]
# Check if this is a Hydrus backend by checking class name
backend_class = type(backend).__name__
if backend_class != "HydrusNetwork":
continue
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is None:
return None
try:
# Query metadata to see if this instance has the file
metadata = backend.get_metadata(hash_str)
if metadata:
return backend_name
except Exception:
# This instance doesn't have the file or had an error
continue
try:
for backend_name, _backend in hydrus_provider.iter_backends():
try:
if hydrus_provider.hash_exists(hash_str, store_name=str(backend_name)):
return str(backend_name)
except Exception:
continue
except Exception:
return None
return None
def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]:
def _find_hydrus_instance_by_url(
url: str,
file_storage: Any,
*,
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Find which Hydrus instance matches a given URL.
Args:
@@ -829,31 +896,13 @@ def _find_hydrus_instance_by_url(url: str, file_storage: Any) -> Optional[str]:
Returns:
Instance name (e.g., 'home') or None if not found
"""
from urllib.parse import urlparse
parsed_target = urlparse(url)
target_netloc = parsed_target.netloc.lower()
# Check each Hydrus backend's URL
for backend_name in file_storage.list_backends():
backend = file_storage[backend_name]
backend_class = type(backend).__name__
if backend_class != "HydrusNetwork":
continue
# Get the backend's base URL from its client
try:
backend_url = backend._client.base_url
parsed_backend = urlparse(backend_url)
backend_netloc = parsed_backend.netloc.lower()
# Match by netloc (host:port)
if target_netloc == backend_netloc:
return backend_name
except Exception:
continue
return None
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is None:
return None
try:
return hydrus_provider.match_store_name_for_url(url)
except Exception:
return None
def _normalize_playlist_path(text: Optional[str]) -> Optional[str]:
@@ -891,7 +940,8 @@ def _normalize_playlist_path(text: Optional[str]) -> Optional[str]:
def _infer_store_from_playlist_item(
item: Dict[str,
Any],
file_storage: Optional[Any] = None
file_storage: Optional[Any] = None,
config: Optional[Dict[str, Any]] = None,
) -> str:
"""Infer a friendly store label from an MPV playlist entry.
@@ -915,7 +965,7 @@ def _infer_store_from_playlist_item(
# If we have file_storage, query each Hydrus instance to find which one has this hash
if file_storage:
hash_str = target.lower()
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
return "hydrus"
@@ -929,7 +979,7 @@ def _infer_store_from_playlist_item(
hash_match = _SHA256_RE.search(target.lower())
if hash_match:
hash_str = hash_match.group(0)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
return "hydrus"
@@ -968,11 +1018,11 @@ def _infer_store_from_playlist_item(
hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match:
hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
# If no hash in URL, try matching the base URL to configured instances
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage)
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
return "hydrus"
@@ -982,10 +1032,10 @@ def _infer_store_from_playlist_item(
hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match:
hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage)
hydrus_instance = _find_hydrus_instance_by_url(target, file_storage, config=config)
if hydrus_instance:
return hydrus_instance
return "hydrus"
@@ -1320,6 +1370,16 @@ def _get_playable_path(
elif isinstance(item, str):
path = item
if not store or not file_hash or str(file_hash).strip().lower() == "unknown":
try:
extracted_store, extracted_hash = _extract_store_and_hash(item, config=config)
except Exception:
extracted_store, extracted_hash = None, None
if extracted_store and not store:
store = extracted_store
if extracted_hash and (not file_hash or str(file_hash).strip().lower() == "unknown"):
file_hash = extracted_hash
# Debug: show incoming values
try:
debug(f"_get_playable_path: store={store}, path={path}, hash={file_hash}")
@@ -1384,6 +1444,7 @@ def _get_playable_path(
# - MPV IPC pipe (transport)
# - PipeObject (pipeline data)
backend_target_resolved = False
hydrus_provider = _get_hydrus_provider(config)
if store and file_hash and file_hash != "unknown" and file_storage:
try:
backend = file_storage[store]
@@ -1391,18 +1452,14 @@ def _get_playable_path(
backend = None
if backend is not None:
backend_class = type(backend).__name__
backend_target_resolved = True
# HydrusNetwork: build a playable API file URL without browser side-effects.
if backend_class == "HydrusNetwork":
# Hydrus playback should resolve via the provider so store aliases and URL building stay centralized.
if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(store)):
try:
client = getattr(backend, "_client", None)
base_url = getattr(client, "url", None)
if base_url:
base_url = str(base_url).rstrip("/")
# Auth is provided via http-header-fields (set in _queue_items).
path = f"{base_url}/get_files/file?hash={file_hash}"
resolved_path = hydrus_provider.build_file_url(file_hash, store_name=str(store))
if resolved_path:
path = resolved_path
except Exception as e:
debug(
f"Error building Hydrus URL from store '{store}': {e}",
@@ -1570,32 +1627,43 @@ def _queue_items(
item_store = None
if isinstance(item, dict):
item_store = item.get("store")
metadata = item.get("full_metadata") or item.get("metadata")
else:
item_store = getattr(item, "store", None)
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not item_store and isinstance(metadata, dict):
item_store = metadata.get("store")
if not item_store:
item_store, _ = _extract_store_and_hash(item, config=config)
if item_store:
item_store_name = str(item_store).strip() or None
if item_store and file_storage:
try:
backend = file_storage[str(item_store)]
except Exception:
backend = None
if backend is not None and type(backend).__name__ == "HydrusNetwork":
client = getattr(backend, "_client", None)
base_url = getattr(client, "url", None)
key = getattr(client, "access_key", None)
if base_url:
effective_hydrus_url = str(base_url).rstrip("/")
if key:
effective_hydrus_header = (
f"Hydrus-Client-API-Access-Key: {str(key).strip()}"
)
effective_ytdl_opts = _build_ytdl_options(
config,
effective_hydrus_header
if item_store_name and file_storage:
hydrus_provider = _get_hydrus_provider(config)
if hydrus_provider is not None:
try:
client = hydrus_provider.get_client(
store_name=item_store_name,
allow_default=False,
)
except Exception:
client = None
if client is not None:
base_url = getattr(client, "url", None)
key = getattr(client, "access_key", None)
if base_url:
effective_hydrus_url = str(base_url).rstrip("/")
if key:
effective_hydrus_header = (
f"Hydrus-Client-API-Access-Key: {str(key).strip()}"
)
effective_ytdl_opts = _build_ytdl_options(
config,
effective_hydrus_header
)
except Exception:
pass
@@ -1608,6 +1676,10 @@ def _queue_items(
f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}"
)
hydrus_target = bool(
effective_hydrus_header and _is_hydrus_path(str(target), effective_hydrus_url)
)
norm_key = _normalize_playlist_path(target) or str(target).strip().lower()
if norm_key in existing_targets or norm_key in new_targets:
debug(f"Skipping duplicate playlist entry: {title or target}")
@@ -1616,10 +1688,11 @@ def _queue_items(
# Use memory:// M3U hack to pass title to MPV.
# Avoid this for probable ytdl URLs because it can prevent the hook from triggering.
if title and not _is_probable_ytdl_url(target):
safe_title = title.replace("\n", " ").replace("\r", "") if title else None
if title and hydrus_target:
target_to_send = target
elif title and not _is_probable_ytdl_url(target):
# Sanitize title for M3U (remove newlines)
safe_title = title.replace("\n", " ").replace("\r", "")
# Carry the store name for hash URLs so MPV.lyric can resolve the backend.
# This is especially important for local file-server URLs like /get_files/file?hash=...
target_for_m3u = target
@@ -1646,15 +1719,14 @@ def _queue_items(
# so MPV.lyric can resolve the correct backend for notes.
if mode == "replace":
try:
s, h = _extract_store_and_hash(item)
s, h = _extract_store_and_hash(item, config=config)
_set_mpv_item_context(s, h)
except Exception:
pass
# If this is a Hydrus path, set header property and yt-dlp headers before loading.
# Use the real target (not the memory:// wrapper) for detection.
if effective_hydrus_header and _is_hydrus_path(str(target),
effective_hydrus_url):
if hydrus_target:
header_cmd = {
"command":
["set_property",
@@ -1683,10 +1755,18 @@ def _queue_items(
except Exception:
pass
load_options: Dict[str, str] = {}
if hydrus_target and command_name == "loadfile":
load_options["ytdl"] = "no"
if safe_title:
load_options["force-media-title"] = safe_title
command_args: List[Any] = [command_name, target_to_send, mode]
if load_options:
command_args.extend([-1, load_options])
cmd = {
"command": [command_name,
target_to_send,
mode],
"command": command_args,
"request_id": 200
}
try:
@@ -2159,7 +2239,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Prefer the store/hash from the piped item when auto-playing.
try:
s, h = _extract_store_and_hash(items_to_add[0])
s, h = _extract_store_and_hash(items_to_add[0], config=config)
_set_mpv_item_context(s, h)
except Exception:
pass
@@ -2293,7 +2373,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
else:
# Play item
try:
s, h = _extract_store_and_hash(item)
s, h = _extract_store_and_hash(item, config=config)
_set_mpv_item_context(s, h)
except Exception:
pass
@@ -2376,26 +2456,17 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Extract the real path/URL from memory:// wrapper if present
real_path = _extract_target_from_memory_uri(filename) or filename
# Try to extract hash from the path/URL
file_hash = None
store_name = None
# Try to extract hash/store from the path/URL via provider-aware normalization.
store_name, file_hash = _extract_store_and_hash(
{
"path": real_path,
"title": title,
},
config=config,
)
# Check if it's a Hydrus URL
if "get_files/file" in real_path or "hash=" in real_path:
# Extract hash from Hydrus URL
hash_match = _HASH_QUERY_RE.search(real_path.lower())
if hash_match:
file_hash = hash_match.group(1)
# Try to find which Hydrus instance has this file
if file_storage:
store_name = _find_hydrus_instance_for_hash(
file_hash,
file_storage
)
if not store_name:
store_name = "hydrus"
# Check if it's a hash-based local file
elif real_path:
if not file_hash and real_path:
# Try to extract hash from filename (e.g., C:\path\1e8c46...a1b2.mp4)
path_obj = Path(real_path)
stem = path_obj.stem # filename without extension
@@ -2407,7 +2478,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if not store_name:
store_name = _infer_store_from_playlist_item(
item,
file_storage=file_storage
file_storage=file_storage,
config=config,
)
# Build PipeObject with proper metadata
@@ -2651,7 +2723,7 @@ def _start_mpv(
# target change (the helper may start before playback begins).
try:
if items:
s, h = _extract_store_and_hash(items[0])
s, h = _extract_store_and_hash(items[0], config=config)
_set_mpv_item_context(s, h)
except Exception:
pass
docs/ftp_plugin_tutorial.md
+33 -23
View File
@@ -2,11 +2,11 @@
This walkthrough adds a real bundled `ftp` plugin so users can:
- run `search-file -plugin ftp ...`
- run `search-file -plugin ftp -instance <name> ...`
- browse remote folders as result tables
- select file rows to `download-file`
- pipe selected file rows into `add-file`
- upload local files with `add-file -plugin ftp`
- upload local files with `add-file -plugin ftp -instance <name>`
The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
@@ -20,14 +20,14 @@ The FTP plugin demonstrates the main provider hooks that matter for a storage-st
- `selector()` turns folder rows into a follow-up table when the user runs `@N`.
- `download()` and `download_url()` fetch FTP files into `download-file` output paths.
- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first.
- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server.
- `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server.
## Example Config
Add an FTP provider block to your config:
Add one or more named FTP provider instances to your config:
```toml
[provider.ftp]
[provider.ftp.work]
host = "ftp.example.com"
port = 21
username = "demo"
@@ -37,11 +37,20 @@ tls = false
passive = true
timeout = 20
search_depth = 1
[provider.ftp.archive]
host = "archive.example.com"
port = 2121
username = "archive-bot"
password = "secret"
base_path = "/dropbox"
tls = true
```
Notes:
- `host` is the only required field for the plugin to validate.
- `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`.
- `host` is the only required field for each instance to validate.
- `username` defaults to `anonymous` and `password` defaults to `anonymous@`.
- `base_path` is both the default search root and the upload target directory.
- `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default.
@@ -51,25 +60,25 @@ Notes:
Basic listing from the configured base path:
```powershell
search-file -plugin ftp "*"
search-file -plugin ftp -instance work "*"
```
Search by filename fragment:
```powershell
search-file -plugin ftp "invoice"
search-file -plugin ftp -instance work "invoice"
```
Search a different subtree and recurse deeper:
```powershell
search-file -plugin ftp "path:/pub depth:2 invoice"
search-file -plugin ftp -instance work "path:/pub depth:2 invoice"
```
Filter to folders only:
```powershell
search-file -plugin ftp "path:/pub type:folder *"
search-file -plugin ftp -instance work "path:/pub type:folder *"
```
The plugin returns rows with explicit columns for name, type, directory, size, and modification time.
@@ -79,20 +88,20 @@ The plugin returns rows with explicit columns for name, type, directory, size, a
Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory:
```powershell
search-file -plugin ftp "*"
search-file -plugin ftp -instance work "*"
@2
```
File rows carry an explicit row action:
```powershell
download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf
download-file -plugin ftp -instance work -url ftp://ftp.example.com/incoming/report.pdf
```
That means plain `@N` on a file row downloads it immediately:
```powershell
search-file -plugin ftp "report"
search-file -plugin ftp -instance work "report"
@1
```
@@ -101,14 +110,14 @@ search-file -plugin ftp "report"
If you want the downloaded file in a specific local directory:
```powershell
search-file -plugin ftp "report"
search-file -plugin ftp -instance work "report"
@1 | download-file -path C:\Downloads
```
If you want to ingest the selected FTP file into a configured store backend:
```powershell
search-file -plugin ftp "report"
search-file -plugin ftp -instance work "report"
@1 | add-file -store tutorial
```
@@ -117,23 +126,24 @@ Why this works:
- the file row advertises a `download-file` row action
- the pipeline auto-inserts that download before `add-file`
- the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion
- file rows also carry the chosen `instance`, so selection replay and `@N | add-file ...` keep the same FTP target
## Upload Flow
Uploading uses the same provider name, but through `add-file -plugin ftp`:
Uploading uses the same provider name, but through `add-file -plugin ftp -instance <name>`:
```powershell
add-file -plugin ftp -path C:\Media\report.pdf
add-file -plugin ftp -instance archive -path C:\Media\report.pdf
```
That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result.
That sends the file to the selected instance's FTP `base_path` and returns the FTP URL as the uploaded result.
## Why The Row Metadata Matters
The critical part of this plugin is the file-row metadata:
- file rows emit `_selection_args` as `['-url', '<ftp-url>']`
- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '<ftp-url>']`
- file rows emit `_selection_args` as `['-instance', '<name>', '-url', '<ftp-url>']`
- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-instance', '<name>', '-url', '<ftp-url>']`
- folder rows do not emit a download action, so `selector()` can own drill-in behavior instead
That split is what keeps these two user experiences compatible:
@@ -155,10 +165,10 @@ The code is intentionally small and uses only Python stdlib pieces:
## Recommended Commands To Demo The Walkthrough
```powershell
search-file -plugin ftp "*"
search-file -plugin ftp "path:/incoming depth:2 *.pdf"
search-file -plugin ftp -instance work "*"
search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -store tutorial
add-file -plugin ftp -path C:\Media\report.pdf
add-file -plugin ftp -instance archive -path C:\Media\report.pdf
```
docs/scp_plugin_tutorial.md
+28 -18
View File
@@ -11,16 +11,16 @@ The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py).
The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
- `search-file -plugin scp ...` lists remote files and folders over SFTP.
- `search-file -plugin scp -instance <name> ...` lists remote files and folders over SFTP.
- plain `@N` on a folder drills into that directory.
- plain `@N` on a file runs `download-file -plugin scp -url ...`.
- plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`.
- `@N | add-file -store ...` downloads first, then ingests the local temp file.
- `add-file -plugin scp -path ...` uploads a local file to the configured remote path.
- `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path.
## Example Config
```toml
[provider.scp]
[provider.scp.work]
host = "ssh.example.com"
port = 22
username = "deploy"
@@ -31,11 +31,20 @@ timeout = 20
search_depth = 1
allow_agent = true
look_for_keys = true
[provider.scp.archive]
host = "ssh-archive.example.com"
port = 2222
username = "archive"
key_path = "C:/Users/Admin/.ssh/archive_ed25519"
base_path = "/srv/archive"
timeout = 20
```
Notes:
- `host` and `username` are required for the plugin to validate.
- `work` and `archive` are instance names; use them with `-instance work` or `-instance archive`.
- `host` and `username` are required for each instance to validate.
- You can use password auth, key auth, or both.
- `base_path` is both the default search root and the default upload directory.
@@ -44,25 +53,25 @@ Notes:
List the configured base path:
```powershell
search-file -plugin scp "*"
search-file -plugin scp -instance work "*"
```
Search by filename:
```powershell
search-file -plugin scp "invoice"
search-file -plugin scp -instance work "invoice"
```
Search another subtree with deeper recursion:
```powershell
search-file -plugin scp "path:/srv/files/releases depth:2 *.zip"
search-file -plugin scp -instance work "path:/srv/files/releases depth:2 *.zip"
```
Show only folders:
```powershell
search-file -plugin scp "path:/srv/files type:folder *"
search-file -plugin scp -instance work "path:/srv/files type:folder *"
```
## Selection Flow
@@ -70,21 +79,21 @@ search-file -plugin scp "path:/srv/files type:folder *"
Folder rows are navigation rows:
```powershell
search-file -plugin scp "*"
search-file -plugin scp -instance work "*"
@2
```
File rows carry an explicit row action, so terminal selection downloads directly:
```powershell
search-file -plugin scp "report"
search-file -plugin scp -instance work "report"
@1
```
That expands to the equivalent of:
```powershell
download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf
download-file -plugin scp -instance work -url scp://ssh.example.com/srv/files/report.pdf
```
## Download And Add-File Flow
@@ -92,14 +101,14 @@ download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf
Download into a local folder:
```powershell
search-file -plugin scp "report"
search-file -plugin scp -instance work "report"
@1 | download-file -path C:\Downloads
```
Ingest a selected remote file into a configured store backend:
```powershell
search-file -plugin scp "report"
search-file -plugin scp -instance work "report"
@1 | add-file -store tutorial
```
@@ -108,13 +117,14 @@ Why this works:
- file rows advertise `_selection_action` for `download-file`
- `add-file` selection replay inserts that provider download stage before ingest
- the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows
- file rows also carry the chosen `instance`, so replay stays bound to the same SSH target
## Upload Flow
Upload a local file to the configured remote `base_path`:
```powershell
add-file -plugin scp -path C:\Media\report.pdf
add-file -plugin scp -instance archive -path C:\Media\report.pdf
```
## Implementation Notes
@@ -127,10 +137,10 @@ The plugin uses SFTP for directory listing because SCP itself is a transfer prot
## Recommended Demo Commands
```powershell
search-file -plugin scp "*"
search-file -plugin scp "path:/srv/files depth:2 *.zip"
search-file -plugin scp -instance work "*"
search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -store tutorial
add-file -plugin scp -path C:\Media\report.pdf
add-file -plugin scp -instance archive -path C:\Media\report.pdf
```
plugins/README.md
+4 -3
View File
@@ -52,8 +52,9 @@ class MyPlugin(Provider):
Bundled walkthrough:
- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`; plugin-mode `-store <name>` remains a compatibility alias while store-backed flows still use real backend stores.
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads.
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives alongside it in [plugins/hydrusnetwork/api.py](plugins/hydrusnetwork/api.py), while [API/HydrusNetwork.py](API/HydrusNetwork.py) remains a compatibility shim. The provider delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
plugins/alldebrid/__init__.py
+77 -3
View File
@@ -540,7 +540,7 @@ def download_magnet(
def expand_folder_item(
item: Any,
get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
get_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
config: Dict[str, Any],
) -> Tuple[List[Any], Optional[str]]:
table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table")
@@ -564,10 +564,10 @@ def expand_folder_item(
except Exception:
magnet_id = None
if magnet_id is None or get_search_plugin is None:
if magnet_id is None or get_plugin is None:
return [], None
plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None
plugin = get_plugin("alldebrid", config) if get_plugin else None
if plugin is None:
return [], None
@@ -1774,6 +1774,80 @@ class AllDebrid(TableProviderMixin, Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
_item, payload, meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if not isinstance(payload, dict):
return False
title = str(payload.get("title") or meta.get("name") or "").strip() or "AllDebrid Item"
magnet_name = str(meta.get("magnet_name") or payload.get("detail") or "").strip()
magnet_id = meta.get("magnet_id")
relpath = str(meta.get("relpath") or "").strip()
direct_url = str(payload.get("path") or "").strip()
selection_url = ""
action = meta.get("_selection_action") or meta.get("selection_action")
if isinstance(action, (list, tuple)):
action_tokens = [str(x) for x in action if x is not None]
for idx, token in enumerate(action_tokens):
if str(token).strip().lower() in {"-url", "--url"} and idx + 1 < len(action_tokens):
selection_url = str(action_tokens[idx + 1] or "").strip()
break
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
payload,
title=title,
store=self.name,
path=direct_url or None,
tags=meta.get("tag") or meta.get("tags"),
extra_fields={
"Plugin": self.name,
"Magnet": magnet_name or None,
"Magnet ID": magnet_id,
"Relative Path": relpath or None,
"View": str(meta.get("provider_view") or meta.get("view") or (table_metadata or {}).get("view") or "").strip() or None,
"Direct Url": direct_url or None,
"Selection Url": selection_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=payload,
title=f"AllDebrid Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Magnet", "Magnet ID", "Relative Path", "View", "Path", "File", "Folder", "ID", "Direct URL", "Selection URL", "Plugin"],
value_case="preserve",
)
try:
from SYS.result_table_adapters import register_plugin
plugins/example_provider.py
+256 -4
View File
@@ -1,7 +1,259 @@
"""Plugin-namespace import shim for the strict adapter example module.
"""Example plugin that uses the new `ResultTable` API.
This keeps example docs pointing at the plugin namespace while the original
implementation remains in ``Provider.example_provider`` for compatibility.
This module demonstrates a minimal provider adapter that yields `ResultModel`
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
(`render_table`) for demonstration.
Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default
"""
from __future__ import annotations
from Provider.example_provider import * # noqa: F401,F403
from pathlib import Path
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column
SAMPLE_ITEMS = [
{
"name": "Book of Awe.pdf",
"path": "sample/Book of Awe.pdf",
"ext": "pdf",
"size": 1024000,
"source": "example",
},
{
"name": "Song of Joy.mp3",
"path": "sample/Song of Joy.mp3",
"ext": "mp3",
"size": 5120000,
"source": "example",
},
{
"name": "Cover Image.jpg",
"path": "sample/Cover Image.jpg",
"ext": "jpg",
"size": 20480,
"source": "example",
},
]
def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]:
"""Convert provider-specific items into `ResultModel` instances.
This adapter enforces the strict API requirement: it yields only
`ResultModel` instances (no legacy dict objects).
"""
for it in items:
title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "")
yield ResultModel(
title=str(title),
path=str(it.get("path")) if it.get("path") else None,
ext=str(it.get("ext")) if it.get("ext") else None,
size_bytes=int(it.get("size")) if it.get("size") is not None else None,
metadata=dict(it),
source=str(it.get("source")) if it.get("source") else "example",
)
# Columns are intentionally *not* mandated. Create a factory that inspects
# sample rows and builds only columns that make sense for the provider data.
from SYS.result_table_api import metadata_column
def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]:
cols: List[ColumnSpec] = [title_column()]
# If any row has an extension, include Ext column
if any(getattr(r, "ext", None) for r in rows):
cols.append(ext_column())
# If any row has size, include Size column
if any(getattr(r, "size_bytes", None) for r in rows):
cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v)))
# Add any top-level metadata keys discovered (up to 3) as optional columns
seen_keys = []
for r in rows:
for k in (r.metadata or {}).keys():
if k in ("name", "title", "path"):
continue
if k not in seen_keys:
seen_keys.append(k)
if len(seen_keys) >= 3:
break
if len(seen_keys) >= 3:
break
for k in seen_keys:
cols.append(metadata_column(k))
return cols
# Selection function: cmdlets rely on this to build selector args when the user
# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer
# -path if available, otherwise fall back to -title.
def selection_fn(row: ResultModel) -> List[str]:
if row.path:
return ["-path", row.path]
return ["-title", row.title]
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
selection_fn=selection_fn,
metadata={"description": "Example provider demonstrating dynamic columns and selectors"},
)
def _format_size(size: Any) -> str:
try:
s = int(size)
except Exception:
return ""
if s >= 1024 ** 3:
return f"{s / (1024 ** 3):.2f} GB"
if s >= 1024 ** 2:
return f"{s / (1024 ** 2):.2f} MB"
if s >= 1024:
return f"{s / 1024:.2f} KB"
return f"{s} B"
def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str:
"""Render a simple ASCII table of `rows` using `columns`.
This is intentionally very small and dependency-free for demonstration.
Renderers in the project should implement the `Renderer` protocol.
"""
rows = list(rows)
# Build cell matrix (strings)
matrix: List[List[str]] = []
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
matrix.append(cells)
# Compute column widths as max(header, content)
headers = [c.header for c in columns]
widths = [len(h) for h in headers]
for row_cells in matrix:
for i, cell in enumerate(row_cells):
widths[i] = max(widths[i], len(cell))
# Helper to format a row
def fmt_row(cells: List[str]) -> str:
return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
lines: List[str] = []
lines.append(fmt_row(headers))
lines.append("-+-".join("-" * w for w in widths))
for row_cells in matrix:
lines.append(fmt_row(row_cells))
return "\n".join(lines)
# Rich-based renderer (returns a Rich Table renderable)
def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]):
"""Render rows as a `rich.table.Table` for terminal output.
Returns the Table object; callers may `Console.print(table)` to render.
"""
try:
from rich.table import Table as RichTable
except Exception as exc: # pragma: no cover - rare if rich missing
raise RuntimeError("rich is required for rich renderer") from exc
table = RichTable(show_header=True, header_style="bold")
for col in columns:
table.add_column(col.header)
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
table.add_row(*cells)
return table
def demo() -> None:
rows = list(adapter(SAMPLE_ITEMS))
table = render_table_rich(rows, columns_factory(rows))
try:
from rich.console import Console
except Exception:
# Fall back to plain printing if rich is not available
print("Example provider output:")
print(render_table(rows, columns_factory(rows)))
return
console = Console()
console.print("Example provider output:")
console.print(table)
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_plugin
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
# Render
try:
from rich.console import Console
except Exception:
print(render_table(rows, cols))
sel_args = provider.selection_args(rows[idx])
print("Selection args for row", idx, "->", sel_args)
return
console = Console()
console.print("Example provider output:")
console.print(render_table_rich(rows, cols))
# Selection args example
sel = provider.selection_args(rows[idx])
console.print("Selection args for row", idx, "->", sel)
if __name__ == "__main__":
demo()
plugins/ftp/__init__.py
+252 -83
View File
@@ -12,18 +12,6 @@ from urllib.parse import quote, unquote, urlparse
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("ftp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
@@ -155,20 +143,54 @@ class FTP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._tls = _coerce_bool(conf.get("tls"), False)
self._port = _coerce_int(conf.get("port"), 21)
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous"
password_value = conf.get("password")
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@"
self._passive = _coerce_bool(conf.get("passive"), True)
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
_instance_name, conf = self.resolve_plugin_instance()
defaults = self._settings_from_config(conf)
self._host = str(defaults.get("host") or "").strip()
self._tls = bool(defaults.get("tls"))
self._port = int(defaults.get("port") or 21)
self._username = str(defaults.get("username") or "anonymous").strip() or "anonymous"
self._password = str(defaults.get("password") or "anonymous@").strip() or "anonymous@"
self._passive = bool(defaults.get("passive"))
self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
password_value = entry.get("password")
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"tls": _coerce_bool(entry.get("tls"), False),
"port": _coerce_int(entry.get("port"), 21),
"username": str(entry.get("username") or entry.get("user") or "anonymous").strip() or "anonymous",
"password": str(password_value).strip() if password_value not in (None, "") else "anonymous@",
"passive": _coerce_bool(entry.get("passive"), True),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool:
return bool(self._host)
settings = self._resolve_settings()
return bool(settings.get("host"))
def config_helper_text(self) -> str:
return "Test the configured FTP/FTPS settings before searching or uploading."
@@ -186,13 +208,14 @@ class FTP(Provider):
if str(action_id or "").strip().lower() != "test_connection":
return super().run_config_action(action_id, **_kwargs)
if not self._host:
settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
ftp = None
try:
ftp = self._connect()
active_path = self._base_path or "/"
ftp = self._connect(settings=settings)
active_path = str(settings.get("base_path") or "/")
try:
ftp.cwd(active_path)
resolved_path = ftp.pwd()
@@ -200,7 +223,7 @@ class FTP(Provider):
resolved_path = active_path
return {
"ok": True,
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.",
"message": f"Connected to FTP {settings.get('host')}:{settings.get('port')} and reached {resolved_path}.",
}
except Exception as exc:
return {"ok": False, "message": f"FTP connection failed: {exc}"}
@@ -211,6 +234,10 @@ class FTP(Provider):
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
@@ -221,17 +248,21 @@ class FTP(Provider):
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip()
if not text or text == "*":
return f"FTP: {active_path}"
return f"FTP: {text} @ {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"instance": settings.get("instance"),
"host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(),
}
@@ -244,15 +275,21 @@ class FTP(Provider):
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
settings = self._resolve_settings(filters=active_filters, require_explicit=True)
if not settings.get("host"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
return self._search_directory(
ftp,
@@ -261,6 +298,7 @@ class FTP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
finally:
self._close(ftp)
@@ -278,19 +316,23 @@ class FTP(Provider):
target_path = ""
target_title = ""
instance_name = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path)
settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path:
break
if not target_path:
return False
ftp = self._connect()
settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ftp = self._connect(settings=settings)
try:
rows = self._search_directory(
ftp,
@@ -299,6 +341,7 @@ class FTP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
finally:
self._close(ftp)
@@ -310,18 +353,23 @@ class FTP(Provider):
return True
title = target_title or target_path
table = Table(f"FTP: {title}")._perseverance(True)
table = Table(f"FTP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("ftp")
try:
table.set_table_metadata({
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"])
source_args = ["-plugin", "ftp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = []
for row in rows:
@@ -329,7 +377,7 @@ class FTP(Provider):
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path})
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
@@ -342,6 +390,77 @@ class FTP(Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "FTP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
ftp_url = str(metadata.get("ftp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("ftp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=ftp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Ftp Url": ftp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"FTP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "FTP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -349,10 +468,15 @@ class FTP(Provider):
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
@@ -365,13 +489,7 @@ class FTP(Provider):
destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename)
ftp = self._connect(
host=settings["host"],
port=settings["port"],
username=settings["username"],
password=settings["password"],
tls=settings["tls"],
)
ftp = self._connect(settings=settings)
try:
with destination.open("wb") as handle:
ftp.retrbinary(f"RETR {remote_path}", handle.write)
@@ -404,7 +522,12 @@ class FTP(Provider):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None:
try:
temp_dir.rmdir()
@@ -424,35 +547,57 @@ class FTP(Provider):
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
raise RuntimeError("No configured FTP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
self._ensure_directory(ftp, remote_dir)
self._ensure_directory(ftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
with local_path.open("rb") as handle:
ftp.storbinary(f"STOR {remote_path}", handle)
finally:
self._close(ftp)
return self._build_url(remote_path)
return self._build_url(remote_path, settings=settings)
def _connect(
self,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
tls: Optional[bool] = None,
) -> ftplib.FTP:
use_tls = self._tls if tls is None else bool(tls)
resolved = dict(settings or {})
use_tls = bool(resolved.get("tls")) if tls is None else bool(tls)
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout)
ftp.login(username or self._username, password or self._password)
ftp.connect(
host or str(resolved.get("host") or self._host),
int(port or resolved.get("port") or self._port),
timeout=max(1, int(resolved.get("timeout") or self._timeout)),
)
ftp.login(
username or str(resolved.get("username") or self._username),
password or str(resolved.get("password") or self._password),
)
try:
ftp.set_pasv(self._passive)
ftp.set_pasv(bool(resolved.get("passive")) if "passive" in resolved else self._passive)
except Exception:
pass
if use_tls and isinstance(ftp, ftplib.FTP_TLS):
@@ -497,32 +642,39 @@ class FTP(Provider):
self,
remote_path: Any,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
tls: Optional[bool] = None,
) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/")
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp"
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
scheme = "ftps" if ((bool(resolved.get("tls")) if tls is None else bool(tls))) else "ftp"
host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "ftp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or settings.get("port") or self._port
username = parsed.username or settings.get("username") or self._username
password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return {
"instance": settings.get("instance"),
"tls": scheme == "ftps",
"host": host,
"port": port,
"username": username,
"password": password,
"path": path_text,
"passive": settings.get("passive", self._passive),
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
}
def _search_directory(
@@ -534,21 +686,22 @@ class FTP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
for entry in self._list_directory(ftp, normalized):
for entry in self._list_directory(ftp, normalized, base_path=str(settings.get("base_path") or self._base_path)):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
@@ -578,16 +731,18 @@ class FTP(Provider):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
ftp_path = str(entry.get("ftp_path") or "/")
ftp_url = self._build_url(ftp_path)
ftp_url = self._build_url(ftp_path, settings=settings)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = {
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"ftp_path": ftp_path,
"ftp_url": ftp_url,
"selection_url": ftp_url,
@@ -599,6 +754,13 @@ class FTP(Provider):
if modified:
metadata["modified"] = modified
selection_args = ["-url", ftp_url]
selection_action = ["download-file", "-plugin", "ftp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", ftp_url])
return SearchResult(
table="ftp",
title=str(entry.get("name") or ftp_path),
@@ -615,13 +777,13 @@ class FTP(Provider):
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", ftp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url],
selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else selection_action,
full_metadata=metadata,
)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=base_path)
try:
entries: List[Dict[str, Any]] = []
for name, facts in ftp.mlsd(normalized):
@@ -716,8 +878,8 @@ class FTP(Provider):
return _format_timestamp(parts[1])
return ""
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/":
return
partial = ""
@@ -763,11 +925,18 @@ class FTP(Provider):
if path_text.startswith(("ftp://", "ftps://")):
ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
if ftp_path:
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path)
base_path = str(metadata.get("base_path") or self._base_path)
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=base_path)
metadata.setdefault("selection_path", metadata["ftp_path"])
if metadata.get("ftp_path") and not metadata.get("ftp_url"):
metadata["ftp_url"] = self._build_url(metadata["ftp_path"])
metadata["ftp_url"] = self._build_url(
metadata["ftp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("ftp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["ftp_url"]
plugins/libgen/__init__.py
+5 -5
View File
@@ -286,7 +286,7 @@ def _enrich_book_tags_from_isbn(isbn: str,
Priority:
1) OpenLibrary API-by-ISBN scrape (fast, structured)
2) isbnsearch.org scrape via MetadataProvider
2) isbnsearch.org scrape via metadata plugin
"""
isbn_clean = re.sub(r"[^0-9Xx]", "", str(isbn or "")).upper()
@@ -381,12 +381,12 @@ def _enrich_book_tags_from_isbn(isbn: str,
except Exception:
pass
# 2) isbnsearch metadata provider fallback.
# 2) isbnsearch metadata plugin fallback.
try:
from plugins.metadata_provider import get_metadata_provider
from plugins.metadata_provider import get_metadata_plugin
provider = get_metadata_provider("isbnsearch",
config or {})
provider = get_metadata_plugin("isbnsearch",
config or {})
if provider is None:
return [], ""
items = provider.search(isbn_clean, limit=1)
plugins/metadata_provider.py
+2029 -5
View File
File diff suppressed because it is too large Load Diff
plugins/scp/__init__.py
+248 -76
View File
@@ -16,18 +16,6 @@ from scp import SCPClient
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("scp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
@@ -166,20 +154,55 @@ class SCP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._port = _coerce_int(conf.get("port"), 22)
self._username = str(conf.get("username") or conf.get("user") or "").strip()
self._password = str(conf.get("password") or "").strip()
self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip()
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._allow_agent = _coerce_bool(conf.get("allow_agent"), True)
self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True)
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
_instance_name, conf = self.resolve_plugin_instance()
defaults = self._settings_from_config(conf)
self._host = str(defaults.get("host") or "").strip()
self._port = int(defaults.get("port") or 22)
self._username = str(defaults.get("username") or "").strip()
self._password = str(defaults.get("password") or "").strip()
self._key_path = str(defaults.get("key_path") or "").strip()
self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._allow_agent = bool(defaults.get("allow_agent"))
self._look_for_keys = bool(defaults.get("look_for_keys"))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"port": _coerce_int(entry.get("port"), 22),
"username": str(entry.get("username") or entry.get("user") or "").strip(),
"password": str(entry.get("password") or "").strip(),
"key_path": str(entry.get("key_path") or entry.get("identity_file") or "").strip(),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"allow_agent": _coerce_bool(entry.get("allow_agent"), True),
"look_for_keys": _coerce_bool(entry.get("look_for_keys"), True),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool:
return bool(self._host and self._username)
settings = self._resolve_settings()
return bool(settings.get("host") and settings.get("username"))
def config_helper_text(self) -> str:
return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here."
@@ -210,6 +233,10 @@ class SCP(Provider):
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
@@ -220,17 +247,21 @@ class SCP(Provider):
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip()
if not text or text == "*":
return f"SCP: {active_path}"
return f"SCP: {text} @ {active_path}"
return f"SCP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"SCP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"instance": settings.get("instance"),
"host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(),
}
@@ -243,15 +274,21 @@ class SCP(Provider):
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
settings = self._resolve_settings(filters=active_filters, require_explicit=True)
if not settings.get("host") or not settings.get("username"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ssh = self._connect_ssh()
ssh = self._connect_ssh(settings)
sftp = None
try:
try:
@@ -266,6 +303,7 @@ class SCP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
return self._search_directory(
@@ -275,6 +313,7 @@ class SCP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
finally:
self._close_client(sftp)
@@ -293,19 +332,23 @@ class SCP(Provider):
target_path = ""
target_title = ""
instance_name = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path)
settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path:
break
if not target_path:
return False
ssh = self._connect_ssh()
settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ssh = self._connect_ssh(settings)
sftp = None
try:
try:
@@ -320,6 +363,7 @@ class SCP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
else:
rows = self._search_directory(
@@ -329,6 +373,7 @@ class SCP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
finally:
self._close_client(sftp)
@@ -341,18 +386,23 @@ class SCP(Provider):
return True
title = target_title or target_path
table = Table(f"SCP: {title}")._perseverance(True)
table = Table(f"SCP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("scp")
try:
table.set_table_metadata({
"provider": "scp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"])
source_args = ["-plugin", "scp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = []
for row in rows:
@@ -360,7 +410,7 @@ class SCP(Provider):
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path})
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
@@ -373,6 +423,77 @@ class SCP(Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "SCP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
scp_url = str(metadata.get("scp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("scp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=scp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Scp Url": scp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"SCP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "SCP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -380,10 +501,15 @@ class SCP(Provider):
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
@@ -431,7 +557,12 @@ class SCP(Provider):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None:
try:
temp_dir.rmdir()
@@ -451,11 +582,24 @@ class SCP(Provider):
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host") or not settings.get("username"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
raise RuntimeError("No configured SCP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ssh = self._connect_ssh()
ssh = self._connect_ssh(settings)
sftp = None
scp_client = None
try:
@@ -466,7 +610,7 @@ class SCP(Provider):
raise
self._ensure_directory_via_ssh(ssh, remote_dir)
else:
self._ensure_directory(sftp, remote_dir)
self._ensure_directory(sftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
scp_client = self._open_scp(ssh)
scp_client.put(str(local_path), remote_path=remote_path)
finally:
@@ -474,19 +618,20 @@ class SCP(Provider):
self._close_client(sftp)
self._close_client(ssh)
return self._build_url(remote_path)
return self._build_url(remote_path, settings=settings)
def _run_test_connection(self) -> Dict[str, Any]:
if not self._host:
settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the SCP connection."}
if not self._username:
if not settings.get("username"):
return {"ok": False, "message": "Set 'username' before testing the SCP connection."}
ssh = None
sftp = None
try:
ssh = self._connect_ssh()
base_path = self._base_path or "/"
ssh = self._connect_ssh(settings)
base_path = str(settings.get("base_path") or "/")
transport_detail = "SFTP available"
try:
sftp = self._open_sftp(ssh)
@@ -502,10 +647,11 @@ class SCP(Provider):
except Exception:
is_dir = False
detail = f" and confirmed {base_path}" if is_dir else ""
auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth"
key_path = str(settings.get("key_path") or "").strip()
auth_mode = f"key {key_path}" if key_path else "password/agent auth"
return {
"ok": True,
"message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.",
"message": f"Connected to SCP {settings.get('host')}:{settings.get('port')} as {settings.get('username')} via {auth_mode}. {transport_detail}{detail}.",
}
except Exception as exc:
return {"ok": False, "message": f"SCP connection failed: {exc}"}
@@ -514,7 +660,9 @@ class SCP(Provider):
self._close_client(ssh)
def _generate_ssh_keypair(self) -> Dict[str, Any]:
target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
settings = self._resolve_settings()
key_path = str(settings.get("key_path") or "").strip()
target = Path(key_path).expanduser() if key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
try:
target.parent.mkdir(parents=True, exist_ok=True)
except Exception as exc:
@@ -530,7 +678,7 @@ class SCP(Provider):
try:
key = paramiko.RSAKey.generate(bits=4096)
key.write_private_key_file(str(target))
comment = f"{self._username or 'medeia'}@{self._host or 'scp'}"
comment = f"{settings.get('username') or 'medeia'}@{settings.get('host') or 'scp'}"
public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8")
try:
target.chmod(0o600)
@@ -654,34 +802,40 @@ class SCP(Provider):
self,
remote_path: Any,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
scheme: str = "scp",
) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/")
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 22 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "scp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or settings.get("port") or self._port
username = parsed.username or settings.get("username") or self._username
password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return {
"instance": settings.get("instance"),
"scheme": scheme,
"host": host,
"port": port,
"username": username,
"password": password,
"key_path": self._key_path,
"allow_agent": self._allow_agent,
"look_for_keys": self._look_for_keys,
"key_path": settings.get("key_path") or self._key_path,
"allow_agent": settings.get("allow_agent", self._allow_agent),
"look_for_keys": settings.get("look_for_keys", self._look_for_keys),
"path": path_text,
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
}
def _search_directory(
@@ -693,12 +847,13 @@ class SCP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
@@ -707,7 +862,7 @@ class SCP(Provider):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("scp_path") or normalized), depth_left - 1)
@@ -723,6 +878,7 @@ class SCP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth)
results: List[SearchResult] = []
@@ -730,7 +886,7 @@ class SCP(Provider):
if len(results) >= limit:
break
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
return results
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
@@ -756,16 +912,18 @@ class SCP(Provider):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
scp_path = str(entry.get("scp_path") or "/")
scp_url = self._build_url(scp_path)
scp_url = self._build_url(scp_path, settings=settings)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(scp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = {
"provider": "scp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"scp_path": scp_path,
"scp_url": scp_url,
"selection_url": scp_url,
@@ -777,6 +935,13 @@ class SCP(Provider):
if modified:
metadata["modified"] = modified
selection_args = ["-url", scp_url]
selection_action = ["download-file", "-plugin", "scp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", scp_url])
return SearchResult(
table="scp",
title=str(entry.get("name") or scp_path),
@@ -793,8 +958,8 @@ class SCP(Provider):
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", scp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url],
selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else selection_action,
full_metadata=metadata,
)
@@ -867,8 +1032,8 @@ class SCP(Provider):
)
return entries
def _ensure_directory(self, sftp: Any, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _ensure_directory(self, sftp: Any, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/":
return
partial = ""
@@ -923,11 +1088,18 @@ class SCP(Provider):
if path_text.startswith(("scp://", "sftp://")):
scp_path = self._normalize_remote_path(path_text, default=self._base_path)
if scp_path:
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path)
base_path = str(metadata.get("base_path") or self._base_path)
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=base_path)
metadata.setdefault("selection_path", metadata["scp_path"])
if metadata.get("scp_path") and not metadata.get("scp_url"):
metadata["scp_url"] = self._build_url(metadata["scp_path"])
metadata["scp_url"] = self._build_url(
metadata["scp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("scp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["scp_url"]
plugins/tidal_manifest.py
+340 -4
View File
@@ -1,7 +1,343 @@
"""Plugin-namespace import shim for Tidal/HIFI manifest helpers.
"""Tidal/HIFI manifest helpers.
The implementation currently lives in ``Provider.tidal_manifest`` while the
legacy namespace is phased out. New imports should prefer ``plugins``.
This module intentionally lives with the provider code (not cmdlets).
It contains best-effort helpers for turning proxy-provided Tidal "manifest"
values into a playable input reference:
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
"""
from Provider.tidal_manifest import * # noqa: F401,F403
from __future__ import annotations
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
_DEFAULT_TIDAL_TRACK_API_BASES = (
"https://triton.squid.wtf",
"https://wolf.qqdl.site",
"https://maus.qqdl.site",
"https://vogel.qqdl.site",
"https://katze.qqdl.site",
"https://hund.qqdl.site",
"https://tidal.kinoplus.online",
"https://tidal-api.binimum.org",
)
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _normalize_api_base(candidate: Any) -> Optional[str]:
text = str(candidate or "").strip()
if not text:
return None
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
return None
return text.rstrip("/")
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
bases: list[str] = []
seen: set[str] = set()
dynamic_candidates = [
metadata.get("_tidal_api_base"),
metadata.get("_api_base"),
metadata.get("api_base"),
metadata.get("base_url"),
]
for candidate in dynamic_candidates:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
return bases
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
except Exception:
return
attempted = False
for base in _iter_track_api_bases(metadata):
attempted = True
track_data: Optional[Dict[str, Any]] = None
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
try:
resp = client.get(
f"{base}/track/",
params=params,
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
track_data = data
break
except Exception:
continue
if isinstance(track_data, dict) and track_data:
try:
metadata.update(track_data)
except Exception:
pass
if not metadata.get("manifest") or not metadata.get("url"):
try:
resp_info = client.get(
f"{base}/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for key, value in info_data.items():
if key not in metadata or not metadata.get(key):
metadata[key] = value
except Exception:
pass
except Exception:
pass
if metadata.get("manifest"):
break
if attempted:
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None