goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updating and refining plugin system refactor

Browse Source
This commit is contained in:
Nose
2026-04-28 22:20:54 -07:00
parent 8685fbb723
commit 323c24f4f4
33 changed files with 4287 additions and 3312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/README.md
+4 -3
View File
@@ -52,8 +52,9 @@ class MyPlugin(Provider):
Bundled walkthrough:
- Providers can now expose named config instances under `provider.<plugin>.<instance>` and cmdlets can target them with `-instance <name>`; plugin-mode `-store <name>` remains a compatibility alias while store-backed flows still use real backend stores.
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads.
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp -instance <name>`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp -instance <name>` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp -instance <name>`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp -instance <name>` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). Its Hydrus client API now lives alongside it in [plugins/hydrusnetwork/api.py](plugins/hydrusnetwork/api.py), while [API/HydrusNetwork.py](API/HydrusNetwork.py) remains a compatibility shim. The provider delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
plugins/alldebrid/__init__.py
+77 -3
View File
@@ -540,7 +540,7 @@ def download_magnet(
def expand_folder_item(
item: Any,
get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
get_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
config: Dict[str, Any],
) -> Tuple[List[Any], Optional[str]]:
table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table")
@@ -564,10 +564,10 @@ def expand_folder_item(
except Exception:
magnet_id = None
if magnet_id is None or get_search_plugin is None:
if magnet_id is None or get_plugin is None:
return [], None
plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None
plugin = get_plugin("alldebrid", config) if get_plugin else None
if plugin is None:
return [], None
@@ -1774,6 +1774,80 @@ class AllDebrid(TableProviderMixin, Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
_item, payload, meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if not isinstance(payload, dict):
return False
title = str(payload.get("title") or meta.get("name") or "").strip() or "AllDebrid Item"
magnet_name = str(meta.get("magnet_name") or payload.get("detail") or "").strip()
magnet_id = meta.get("magnet_id")
relpath = str(meta.get("relpath") or "").strip()
direct_url = str(payload.get("path") or "").strip()
selection_url = ""
action = meta.get("_selection_action") or meta.get("selection_action")
if isinstance(action, (list, tuple)):
action_tokens = [str(x) for x in action if x is not None]
for idx, token in enumerate(action_tokens):
if str(token).strip().lower() in {"-url", "--url"} and idx + 1 < len(action_tokens):
selection_url = str(action_tokens[idx + 1] or "").strip()
break
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
payload,
title=title,
store=self.name,
path=direct_url or None,
tags=meta.get("tag") or meta.get("tags"),
extra_fields={
"Plugin": self.name,
"Magnet": magnet_name or None,
"Magnet ID": magnet_id,
"Relative Path": relpath or None,
"View": str(meta.get("provider_view") or meta.get("view") or (table_metadata or {}).get("view") or "").strip() or None,
"Direct Url": direct_url or None,
"Selection Url": selection_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=payload,
title=f"AllDebrid Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Magnet", "Magnet ID", "Relative Path", "View", "Path", "File", "Folder", "ID", "Direct URL", "Selection URL", "Plugin"],
value_case="preserve",
)
try:
from SYS.result_table_adapters import register_plugin
plugins/example_provider.py
+256 -4
View File
@@ -1,7 +1,259 @@
"""Plugin-namespace import shim for the strict adapter example module.
"""Example plugin that uses the new `ResultTable` API.
This keeps example docs pointing at the plugin namespace while the original
implementation remains in ``Provider.example_provider`` for compatibility.
This module demonstrates a minimal provider adapter that yields `ResultModel`
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
(`render_table`) for demonstration.
Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
plugin-table -plugin example -sample | select -select 1 | add-file -store default
"""
from __future__ import annotations
from Provider.example_provider import * # noqa: F401,F403
from pathlib import Path
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ColumnSpec, ResultModel, title_column, ext_column
SAMPLE_ITEMS = [
{
"name": "Book of Awe.pdf",
"path": "sample/Book of Awe.pdf",
"ext": "pdf",
"size": 1024000,
"source": "example",
},
{
"name": "Song of Joy.mp3",
"path": "sample/Song of Joy.mp3",
"ext": "mp3",
"size": 5120000,
"source": "example",
},
{
"name": "Cover Image.jpg",
"path": "sample/Cover Image.jpg",
"ext": "jpg",
"size": 20480,
"source": "example",
},
]
def adapter(items: Iterable[Dict[str, Any]]) -> Iterable[ResultModel]:
"""Convert provider-specific items into `ResultModel` instances.
This adapter enforces the strict API requirement: it yields only
`ResultModel` instances (no legacy dict objects).
"""
for it in items:
title = it.get("name") or it.get("title") or (Path(str(it.get("path"))).stem if it.get("path") else "")
yield ResultModel(
title=str(title),
path=str(it.get("path")) if it.get("path") else None,
ext=str(it.get("ext")) if it.get("ext") else None,
size_bytes=int(it.get("size")) if it.get("size") is not None else None,
metadata=dict(it),
source=str(it.get("source")) if it.get("source") else "example",
)
# Columns are intentionally *not* mandated. Create a factory that inspects
# sample rows and builds only columns that make sense for the provider data.
from SYS.result_table_api import metadata_column
def columns_factory(rows: List[ResultModel]) -> List[ColumnSpec]:
cols: List[ColumnSpec] = [title_column()]
# If any row has an extension, include Ext column
if any(getattr(r, "ext", None) for r in rows):
cols.append(ext_column())
# If any row has size, include Size column
if any(getattr(r, "size_bytes", None) for r in rows):
cols.append(ColumnSpec("size", "Size", lambda rr: rr.size_bytes or "", lambda v: _format_size(v)))
# Add any top-level metadata keys discovered (up to 3) as optional columns
seen_keys = []
for r in rows:
for k in (r.metadata or {}).keys():
if k in ("name", "title", "path"):
continue
if k not in seen_keys:
seen_keys.append(k)
if len(seen_keys) >= 3:
break
if len(seen_keys) >= 3:
break
for k in seen_keys:
cols.append(metadata_column(k))
return cols
# Selection function: cmdlets rely on this to build selector args when the user
# selects a row (e.g., '@3' -> run next-cmd with the returned args). Prefer
# -path if available, otherwise fall back to -title.
def selection_fn(row: ResultModel) -> List[str]:
if row.path:
return ["-path", row.path]
return ["-title", row.title]
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
selection_fn=selection_fn,
metadata={"description": "Example provider demonstrating dynamic columns and selectors"},
)
def _format_size(size: Any) -> str:
try:
s = int(size)
except Exception:
return ""
if s >= 1024 ** 3:
return f"{s / (1024 ** 3):.2f} GB"
if s >= 1024 ** 2:
return f"{s / (1024 ** 2):.2f} MB"
if s >= 1024:
return f"{s / 1024:.2f} KB"
return f"{s} B"
def render_table(rows: Iterable[ResultModel], columns: List[ColumnSpec]) -> str:
"""Render a simple ASCII table of `rows` using `columns`.
This is intentionally very small and dependency-free for demonstration.
Renderers in the project should implement the `Renderer` protocol.
"""
rows = list(rows)
# Build cell matrix (strings)
matrix: List[List[str]] = []
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
matrix.append(cells)
# Compute column widths as max(header, content)
headers = [c.header for c in columns]
widths = [len(h) for h in headers]
for row_cells in matrix:
for i, cell in enumerate(row_cells):
widths[i] = max(widths[i], len(cell))
# Helper to format a row
def fmt_row(cells: List[str]) -> str:
return " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
lines: List[str] = []
lines.append(fmt_row(headers))
lines.append("-+-".join("-" * w for w in widths))
for row_cells in matrix:
lines.append(fmt_row(row_cells))
return "\n".join(lines)
# Rich-based renderer (returns a Rich Table renderable)
def render_table_rich(rows: Iterable[ResultModel], columns: List[ColumnSpec]):
"""Render rows as a `rich.table.Table` for terminal output.
Returns the Table object; callers may `Console.print(table)` to render.
"""
try:
from rich.table import Table as RichTable
except Exception as exc: # pragma: no cover - rare if rich missing
raise RuntimeError("rich is required for rich renderer") from exc
table = RichTable(show_header=True, header_style="bold")
for col in columns:
table.add_column(col.header)
for r in rows:
cells: List[str] = []
for col in columns:
raw = col.extractor(r)
if col.format_fn:
try:
cell = col.format_fn(raw)
except Exception:
cell = str(raw or "")
else:
cell = str(raw or "")
cells.append(cell)
table.add_row(*cells)
return table
def demo() -> None:
rows = list(adapter(SAMPLE_ITEMS))
table = render_table_rich(rows, columns_factory(rows))
try:
from rich.console import Console
except Exception:
# Fall back to plain printing if rich is not available
print("Example provider output:")
print(render_table(rows, columns_factory(rows)))
return
console = Console()
console.print("Example provider output:")
console.print(table)
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_plugin
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
# Render
try:
from rich.console import Console
except Exception:
print(render_table(rows, cols))
sel_args = provider.selection_args(rows[idx])
print("Selection args for row", idx, "->", sel_args)
return
console = Console()
console.print("Example provider output:")
console.print(render_table_rich(rows, cols))
# Selection args example
sel = provider.selection_args(rows[idx])
console.print("Selection args for row", idx, "->", sel)
if __name__ == "__main__":
demo()
plugins/ftp/__init__.py
+252 -83
View File
@@ -12,18 +12,6 @@ from urllib.parse import quote, unquote, urlparse
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("ftp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
@@ -155,20 +143,54 @@ class FTP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._tls = _coerce_bool(conf.get("tls"), False)
self._port = _coerce_int(conf.get("port"), 21)
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous"
password_value = conf.get("password")
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@"
self._passive = _coerce_bool(conf.get("passive"), True)
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
_instance_name, conf = self.resolve_plugin_instance()
defaults = self._settings_from_config(conf)
self._host = str(defaults.get("host") or "").strip()
self._tls = bool(defaults.get("tls"))
self._port = int(defaults.get("port") or 21)
self._username = str(defaults.get("username") or "anonymous").strip() or "anonymous"
self._password = str(defaults.get("password") or "anonymous@").strip() or "anonymous@"
self._passive = bool(defaults.get("passive"))
self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
password_value = entry.get("password")
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"tls": _coerce_bool(entry.get("tls"), False),
"port": _coerce_int(entry.get("port"), 21),
"username": str(entry.get("username") or entry.get("user") or "anonymous").strip() or "anonymous",
"password": str(password_value).strip() if password_value not in (None, "") else "anonymous@",
"passive": _coerce_bool(entry.get("passive"), True),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool:
return bool(self._host)
settings = self._resolve_settings()
return bool(settings.get("host"))
def config_helper_text(self) -> str:
return "Test the configured FTP/FTPS settings before searching or uploading."
@@ -186,13 +208,14 @@ class FTP(Provider):
if str(action_id or "").strip().lower() != "test_connection":
return super().run_config_action(action_id, **_kwargs)
if not self._host:
settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
ftp = None
try:
ftp = self._connect()
active_path = self._base_path or "/"
ftp = self._connect(settings=settings)
active_path = str(settings.get("base_path") or "/")
try:
ftp.cwd(active_path)
resolved_path = ftp.pwd()
@@ -200,7 +223,7 @@ class FTP(Provider):
resolved_path = active_path
return {
"ok": True,
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.",
"message": f"Connected to FTP {settings.get('host')}:{settings.get('port')} and reached {resolved_path}.",
}
except Exception as exc:
return {"ok": False, "message": f"FTP connection failed: {exc}"}
@@ -211,6 +234,10 @@ class FTP(Provider):
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
@@ -221,17 +248,21 @@ class FTP(Provider):
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip()
if not text or text == "*":
return f"FTP: {active_path}"
return f"FTP: {text} @ {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"instance": settings.get("instance"),
"host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(),
}
@@ -244,15 +275,21 @@ class FTP(Provider):
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
settings = self._resolve_settings(filters=active_filters, require_explicit=True)
if not settings.get("host"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
return self._search_directory(
ftp,
@@ -261,6 +298,7 @@ class FTP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
finally:
self._close(ftp)
@@ -278,19 +316,23 @@ class FTP(Provider):
target_path = ""
target_title = ""
instance_name = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path)
settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path:
break
if not target_path:
return False
ftp = self._connect()
settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ftp = self._connect(settings=settings)
try:
rows = self._search_directory(
ftp,
@@ -299,6 +341,7 @@ class FTP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
finally:
self._close(ftp)
@@ -310,18 +353,23 @@ class FTP(Provider):
return True
title = target_title or target_path
table = Table(f"FTP: {title}")._perseverance(True)
table = Table(f"FTP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("ftp")
try:
table.set_table_metadata({
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"])
source_args = ["-plugin", "ftp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = []
for row in rows:
@@ -329,7 +377,7 @@ class FTP(Provider):
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path})
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
@@ -342,6 +390,77 @@ class FTP(Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "FTP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
ftp_url = str(metadata.get("ftp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("ftp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=ftp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Ftp Url": ftp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"FTP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "FTP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -349,10 +468,15 @@ class FTP(Provider):
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
@@ -365,13 +489,7 @@ class FTP(Provider):
destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename)
ftp = self._connect(
host=settings["host"],
port=settings["port"],
username=settings["username"],
password=settings["password"],
tls=settings["tls"],
)
ftp = self._connect(settings=settings)
try:
with destination.open("wb") as handle:
ftp.retrbinary(f"RETR {remote_path}", handle.write)
@@ -404,7 +522,12 @@ class FTP(Provider):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None:
try:
temp_dir.rmdir()
@@ -424,35 +547,57 @@ class FTP(Provider):
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
raise RuntimeError("No configured FTP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
self._ensure_directory(ftp, remote_dir)
self._ensure_directory(ftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
with local_path.open("rb") as handle:
ftp.storbinary(f"STOR {remote_path}", handle)
finally:
self._close(ftp)
return self._build_url(remote_path)
return self._build_url(remote_path, settings=settings)
def _connect(
self,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
tls: Optional[bool] = None,
) -> ftplib.FTP:
use_tls = self._tls if tls is None else bool(tls)
resolved = dict(settings or {})
use_tls = bool(resolved.get("tls")) if tls is None else bool(tls)
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout)
ftp.login(username or self._username, password or self._password)
ftp.connect(
host or str(resolved.get("host") or self._host),
int(port or resolved.get("port") or self._port),
timeout=max(1, int(resolved.get("timeout") or self._timeout)),
)
ftp.login(
username or str(resolved.get("username") or self._username),
password or str(resolved.get("password") or self._password),
)
try:
ftp.set_pasv(self._passive)
ftp.set_pasv(bool(resolved.get("passive")) if "passive" in resolved else self._passive)
except Exception:
pass
if use_tls and isinstance(ftp, ftplib.FTP_TLS):
@@ -497,32 +642,39 @@ class FTP(Provider):
self,
remote_path: Any,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
tls: Optional[bool] = None,
) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/")
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp"
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
scheme = "ftps" if ((bool(resolved.get("tls")) if tls is None else bool(tls))) else "ftp"
host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "ftp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or settings.get("port") or self._port
username = parsed.username or settings.get("username") or self._username
password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return {
"instance": settings.get("instance"),
"tls": scheme == "ftps",
"host": host,
"port": port,
"username": username,
"password": password,
"path": path_text,
"passive": settings.get("passive", self._passive),
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
}
def _search_directory(
@@ -534,21 +686,22 @@ class FTP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
for entry in self._list_directory(ftp, normalized):
for entry in self._list_directory(ftp, normalized, base_path=str(settings.get("base_path") or self._base_path)):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
@@ -578,16 +731,18 @@ class FTP(Provider):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
ftp_path = str(entry.get("ftp_path") or "/")
ftp_url = self._build_url(ftp_path)
ftp_url = self._build_url(ftp_path, settings=settings)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = {
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"ftp_path": ftp_path,
"ftp_url": ftp_url,
"selection_url": ftp_url,
@@ -599,6 +754,13 @@ class FTP(Provider):
if modified:
metadata["modified"] = modified
selection_args = ["-url", ftp_url]
selection_action = ["download-file", "-plugin", "ftp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", ftp_url])
return SearchResult(
table="ftp",
title=str(entry.get("name") or ftp_path),
@@ -615,13 +777,13 @@ class FTP(Provider):
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", ftp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url],
selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else selection_action,
full_metadata=metadata,
)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=base_path)
try:
entries: List[Dict[str, Any]] = []
for name, facts in ftp.mlsd(normalized):
@@ -716,8 +878,8 @@ class FTP(Provider):
return _format_timestamp(parts[1])
return ""
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/":
return
partial = ""
@@ -763,11 +925,18 @@ class FTP(Provider):
if path_text.startswith(("ftp://", "ftps://")):
ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
if ftp_path:
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path)
base_path = str(metadata.get("base_path") or self._base_path)
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=base_path)
metadata.setdefault("selection_path", metadata["ftp_path"])
if metadata.get("ftp_path") and not metadata.get("ftp_url"):
metadata["ftp_url"] = self._build_url(metadata["ftp_path"])
metadata["ftp_url"] = self._build_url(
metadata["ftp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("ftp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["ftp_url"]
plugins/libgen/__init__.py
+5 -5
View File
@@ -286,7 +286,7 @@ def _enrich_book_tags_from_isbn(isbn: str,
Priority:
1) OpenLibrary API-by-ISBN scrape (fast, structured)
2) isbnsearch.org scrape via MetadataProvider
2) isbnsearch.org scrape via metadata plugin
"""
isbn_clean = re.sub(r"[^0-9Xx]", "", str(isbn or "")).upper()
@@ -381,12 +381,12 @@ def _enrich_book_tags_from_isbn(isbn: str,
except Exception:
pass
# 2) isbnsearch metadata provider fallback.
# 2) isbnsearch metadata plugin fallback.
try:
from plugins.metadata_provider import get_metadata_provider
from plugins.metadata_provider import get_metadata_plugin
provider = get_metadata_provider("isbnsearch",
config or {})
provider = get_metadata_plugin("isbnsearch",
config or {})
if provider is None:
return [], ""
items = provider.search(isbn_clean, limit=1)
plugins/metadata_provider.py
+2029 -5
View File
File diff suppressed because it is too large Load Diff
plugins/scp/__init__.py
+248 -76
View File
@@ -16,18 +16,6 @@ from scp import SCPClient
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("scp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
@@ -166,20 +154,55 @@ class SCP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._port = _coerce_int(conf.get("port"), 22)
self._username = str(conf.get("username") or conf.get("user") or "").strip()
self._password = str(conf.get("password") or "").strip()
self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip()
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._allow_agent = _coerce_bool(conf.get("allow_agent"), True)
self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True)
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
_instance_name, conf = self.resolve_plugin_instance()
defaults = self._settings_from_config(conf)
self._host = str(defaults.get("host") or "").strip()
self._port = int(defaults.get("port") or 22)
self._username = str(defaults.get("username") or "").strip()
self._password = str(defaults.get("password") or "").strip()
self._key_path = str(defaults.get("key_path") or "").strip()
self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._allow_agent = bool(defaults.get("allow_agent"))
self._look_for_keys = bool(defaults.get("look_for_keys"))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"port": _coerce_int(entry.get("port"), 22),
"username": str(entry.get("username") or entry.get("user") or "").strip(),
"password": str(entry.get("password") or "").strip(),
"key_path": str(entry.get("key_path") or entry.get("identity_file") or "").strip(),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"allow_agent": _coerce_bool(entry.get("allow_agent"), True),
"look_for_keys": _coerce_bool(entry.get("look_for_keys"), True),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool:
return bool(self._host and self._username)
settings = self._resolve_settings()
return bool(settings.get("host") and settings.get("username"))
def config_helper_text(self) -> str:
return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here."
@@ -210,6 +233,10 @@ class SCP(Provider):
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
@@ -220,17 +247,21 @@ class SCP(Provider):
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip()
if not text or text == "*":
return f"SCP: {active_path}"
return f"SCP: {text} @ {active_path}"
return f"SCP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"SCP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"instance": settings.get("instance"),
"host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(),
}
@@ -243,15 +274,21 @@ class SCP(Provider):
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
settings = self._resolve_settings(filters=active_filters, require_explicit=True)
if not settings.get("host") or not settings.get("username"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ssh = self._connect_ssh()
ssh = self._connect_ssh(settings)
sftp = None
try:
try:
@@ -266,6 +303,7 @@ class SCP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
return self._search_directory(
@@ -275,6 +313,7 @@ class SCP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
finally:
self._close_client(sftp)
@@ -293,19 +332,23 @@ class SCP(Provider):
target_path = ""
target_title = ""
instance_name = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path)
settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path:
break
if not target_path:
return False
ssh = self._connect_ssh()
settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ssh = self._connect_ssh(settings)
sftp = None
try:
try:
@@ -320,6 +363,7 @@ class SCP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
else:
rows = self._search_directory(
@@ -329,6 +373,7 @@ class SCP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
finally:
self._close_client(sftp)
@@ -341,18 +386,23 @@ class SCP(Provider):
return True
title = target_title or target_path
table = Table(f"SCP: {title}")._perseverance(True)
table = Table(f"SCP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("scp")
try:
table.set_table_metadata({
"provider": "scp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"])
source_args = ["-plugin", "scp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = []
for row in rows:
@@ -360,7 +410,7 @@ class SCP(Provider):
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path})
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
@@ -373,6 +423,77 @@ class SCP(Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "SCP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
scp_url = str(metadata.get("scp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("scp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=scp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Scp Url": scp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"SCP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "SCP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -380,10 +501,15 @@ class SCP(Provider):
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
@@ -431,7 +557,12 @@ class SCP(Provider):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None:
try:
temp_dir.rmdir()
@@ -451,11 +582,24 @@ class SCP(Provider):
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host") or not settings.get("username"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"SCP instance '{requested}' is unavailable")
raise RuntimeError("No configured SCP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ssh = self._connect_ssh()
ssh = self._connect_ssh(settings)
sftp = None
scp_client = None
try:
@@ -466,7 +610,7 @@ class SCP(Provider):
raise
self._ensure_directory_via_ssh(ssh, remote_dir)
else:
self._ensure_directory(sftp, remote_dir)
self._ensure_directory(sftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
scp_client = self._open_scp(ssh)
scp_client.put(str(local_path), remote_path=remote_path)
finally:
@@ -474,19 +618,20 @@ class SCP(Provider):
self._close_client(sftp)
self._close_client(ssh)
return self._build_url(remote_path)
return self._build_url(remote_path, settings=settings)
def _run_test_connection(self) -> Dict[str, Any]:
if not self._host:
settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the SCP connection."}
if not self._username:
if not settings.get("username"):
return {"ok": False, "message": "Set 'username' before testing the SCP connection."}
ssh = None
sftp = None
try:
ssh = self._connect_ssh()
base_path = self._base_path or "/"
ssh = self._connect_ssh(settings)
base_path = str(settings.get("base_path") or "/")
transport_detail = "SFTP available"
try:
sftp = self._open_sftp(ssh)
@@ -502,10 +647,11 @@ class SCP(Provider):
except Exception:
is_dir = False
detail = f" and confirmed {base_path}" if is_dir else ""
auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth"
key_path = str(settings.get("key_path") or "").strip()
auth_mode = f"key {key_path}" if key_path else "password/agent auth"
return {
"ok": True,
"message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.",
"message": f"Connected to SCP {settings.get('host')}:{settings.get('port')} as {settings.get('username')} via {auth_mode}. {transport_detail}{detail}.",
}
except Exception as exc:
return {"ok": False, "message": f"SCP connection failed: {exc}"}
@@ -514,7 +660,9 @@ class SCP(Provider):
self._close_client(ssh)
def _generate_ssh_keypair(self) -> Dict[str, Any]:
target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
settings = self._resolve_settings()
key_path = str(settings.get("key_path") or "").strip()
target = Path(key_path).expanduser() if key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
try:
target.parent.mkdir(parents=True, exist_ok=True)
except Exception as exc:
@@ -530,7 +678,7 @@ class SCP(Provider):
try:
key = paramiko.RSAKey.generate(bits=4096)
key.write_private_key_file(str(target))
comment = f"{self._username or 'medeia'}@{self._host or 'scp'}"
comment = f"{settings.get('username') or 'medeia'}@{settings.get('host') or 'scp'}"
public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8")
try:
target.chmod(0o600)
@@ -654,34 +802,40 @@ class SCP(Provider):
self,
remote_path: Any,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
scheme: str = "scp",
) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/")
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 22 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "scp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or settings.get("port") or self._port
username = parsed.username or settings.get("username") or self._username
password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return {
"instance": settings.get("instance"),
"scheme": scheme,
"host": host,
"port": port,
"username": username,
"password": password,
"key_path": self._key_path,
"allow_agent": self._allow_agent,
"look_for_keys": self._look_for_keys,
"key_path": settings.get("key_path") or self._key_path,
"allow_agent": settings.get("allow_agent", self._allow_agent),
"look_for_keys": settings.get("look_for_keys", self._look_for_keys),
"path": path_text,
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
}
def _search_directory(
@@ -693,12 +847,13 @@ class SCP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
@@ -707,7 +862,7 @@ class SCP(Provider):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("scp_path") or normalized), depth_left - 1)
@@ -723,6 +878,7 @@ class SCP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth)
results: List[SearchResult] = []
@@ -730,7 +886,7 @@ class SCP(Provider):
if len(results) >= limit:
break
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
return results
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
@@ -756,16 +912,18 @@ class SCP(Provider):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
scp_path = str(entry.get("scp_path") or "/")
scp_url = self._build_url(scp_path)
scp_url = self._build_url(scp_path, settings=settings)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(scp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = {
"provider": "scp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"scp_path": scp_path,
"scp_url": scp_url,
"selection_url": scp_url,
@@ -777,6 +935,13 @@ class SCP(Provider):
if modified:
metadata["modified"] = modified
selection_args = ["-url", scp_url]
selection_action = ["download-file", "-plugin", "scp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", scp_url])
return SearchResult(
table="scp",
title=str(entry.get("name") or scp_path),
@@ -793,8 +958,8 @@ class SCP(Provider):
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", scp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url],
selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else selection_action,
full_metadata=metadata,
)
@@ -867,8 +1032,8 @@ class SCP(Provider):
)
return entries
def _ensure_directory(self, sftp: Any, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _ensure_directory(self, sftp: Any, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/":
return
partial = ""
@@ -923,11 +1088,18 @@ class SCP(Provider):
if path_text.startswith(("scp://", "sftp://")):
scp_path = self._normalize_remote_path(path_text, default=self._base_path)
if scp_path:
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path)
base_path = str(metadata.get("base_path") or self._base_path)
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=base_path)
metadata.setdefault("selection_path", metadata["scp_path"])
if metadata.get("scp_path") and not metadata.get("scp_url"):
metadata["scp_url"] = self._build_url(metadata["scp_path"])
metadata["scp_url"] = self._build_url(
metadata["scp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("scp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["scp_url"]
plugins/tidal_manifest.py
+340 -4
View File
@@ -1,7 +1,343 @@
"""Plugin-namespace import shim for Tidal/HIFI manifest helpers.
"""Tidal/HIFI manifest helpers.
The implementation currently lives in ``Provider.tidal_manifest`` while the
legacy namespace is phased out. New imports should prefer ``plugins``.
This module intentionally lives with the provider code (not cmdlets).
It contains best-effort helpers for turning proxy-provided Tidal "manifest"
values into a playable input reference:
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
"""
from Provider.tidal_manifest import * # noqa: F401,F403
from __future__ import annotations
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
_DEFAULT_TIDAL_TRACK_API_BASES = (
"https://triton.squid.wtf",
"https://wolf.qqdl.site",
"https://maus.qqdl.site",
"https://vogel.qqdl.site",
"https://katze.qqdl.site",
"https://hund.qqdl.site",
"https://tidal.kinoplus.online",
"https://tidal-api.binimum.org",
)
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _normalize_api_base(candidate: Any) -> Optional[str]:
text = str(candidate or "").strip()
if not text:
return None
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
return None
return text.rstrip("/")
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
bases: list[str] = []
seen: set[str] = set()
dynamic_candidates = [
metadata.get("_tidal_api_base"),
metadata.get("_api_base"),
metadata.get("api_base"),
metadata.get("base_url"),
]
for candidate in dynamic_candidates:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
normalized = _normalize_api_base(candidate)
if normalized and normalized not in seen:
seen.add(normalized)
bases.append(normalized)
return bases
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
except Exception:
return
attempted = False
for base in _iter_track_api_bases(metadata):
attempted = True
track_data: Optional[Dict[str, Any]] = None
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
try:
resp = client.get(
f"{base}/track/",
params=params,
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
track_data = data
break
except Exception:
continue
if isinstance(track_data, dict) and track_data:
try:
metadata.update(track_data)
except Exception:
pass
if not metadata.get("manifest") or not metadata.get("url"):
try:
resp_info = client.get(
f"{base}/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for key, value in info_data.items():
if key not in metadata or not metadata.get(key):
metadata[key] = value
except Exception:
pass
except Exception:
pass
if metadata.get("manifest"):
break
if attempted:
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None