goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

huge refactor of the entire codebase, with the goal of improving maintainability, readability, and extensibility. This commit includes changes to almost every file in the project, including:

Browse Source
This commit is contained in:
Nose
2026-04-19 00:41:09 -07:00
parent d9e736172a
commit bafd37fdfb
50 changed files with 3258 additions and 4177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
API/data/alldebrid.json
+5 -5
View File
@@ -37,7 +37,7 @@
"(rapidgator\\.net/file/[0-9]{7,8})"
],
"regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))",
"status": true
"status": false
},
"turbobit": {
"name": "turbobit",
@@ -425,7 +425,7 @@
"(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})"
],
"regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})",
"status": false
"status": true
},
"hot4share": {
"name": "hot4share",
@@ -482,7 +482,7 @@
"(katfile\\.com/[0-9a-zA-Z]{12})"
],
"regexp": "(katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))",
"status": false
"status": true
},
"mediafire": {
"name": "mediafire",
@@ -595,7 +595,7 @@
"(simfileshare\\.net/download/[0-9]+/)"
],
"regexp": "(simfileshare\\.net/download/[0-9]+/)",
"status": false
"status": true
},
"streamtape": {
"name": "streamtape",
@@ -690,7 +690,7 @@
"uploadrar\\.(net|com)/([0-9a-z]{12})"
],
"regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))",
"status": false,
"status": true,
"hardRedirect": [
"uploadrar.com/([0-9a-zA-Z]{12})"
]
CLI.py
+21 -202
View File
@@ -59,13 +59,9 @@ from SYS.rich_display import (
)
from cmdnat._status_shared import (
add_startup_check as _shared_add_startup_check,
default_provider_ping_targets as _default_provider_ping_targets,
has_provider as _has_provider,
collect_plugin_startup_checks as _collect_plugin_startup_checks,
has_store_subtype as _has_store_subtype,
has_tool as _has_tool,
ping_first as _ping_first,
ping_url as _ping_url,
provider_display_name as _provider_display_name,
)
@@ -98,12 +94,12 @@ from SYS.cmdlet_catalog import (
list_cmdlet_metadata,
list_cmdlet_names,
)
from SYS.config import load_config
from SYS.config import load_config, resolve_cookies_path
from SYS.result_table import Table
from SYS.worker import WorkerManagerRegistry, WorkerStages, WorkerOutputMirror, WorkerStageSession
from SYS.pipeline import PipelineExecutor
from ProviderCore.registry import provider_inline_query_choices
from ProviderCore.registry import plugin_inline_query_choices
@@ -507,25 +503,25 @@ class CmdletIntrospection:
if backends:
return backends
if normalized_arg == "provider":
if normalized_arg == "plugin":
canonical_cmd = (cmd_name or "").replace("_", "-").lower()
try:
from ProviderCore.registry import list_search_providers, list_file_providers
from ProviderCore.registry import list_search_plugins, list_upload_plugins
except Exception:
list_search_providers = None # type: ignore
list_file_providers = None # type: ignore
list_search_plugins = None # type: ignore
list_upload_plugins = None # type: ignore
provider_choices: List[str] = []
if canonical_cmd in {"add-file"} and list_file_providers is not None:
providers = list_file_providers(config) or {}
if canonical_cmd in {"add-file"} and list_upload_plugins is not None:
providers = list_upload_plugins(config) or {}
available = [
name for name, is_ready in providers.items() if is_ready
]
return sorted(available) if available else sorted(providers.keys())
if list_search_providers is not None:
providers = list_search_providers(config) or {}
if list_search_plugins is not None:
providers = list_search_plugins(config) or {}
available = [
name for name, is_ready in providers.items() if is_ready
]
@@ -680,7 +676,7 @@ class CmdletCompleter(Completer):
provider_name = None
if cmd_name == "search-file":
provider_name = self._flag_value(stage_tokens, "-provider", "--provider")
provider_name = self._flag_value(stage_tokens, "-plugin", "--plugin")
if (
cmd_name == "search-file"
@@ -705,7 +701,7 @@ class CmdletCompleter(Completer):
field, partial = inline_token.split(":", 1)
field = field.strip().lower()
partial_lower = partial.strip().lower()
inline_choices = provider_inline_query_choices(provider_name, field, config)
inline_choices = plugin_inline_query_choices(provider_name, field, config)
if inline_choices:
filtered = (
[c for c in inline_choices if partial_lower in str(c).lower()]
@@ -728,7 +724,7 @@ class CmdletCompleter(Completer):
if choices:
choice_list = choices
normalized_prev = prev_token.lstrip("-").strip().lower()
if normalized_prev == "provider" and current_token:
if normalized_prev in {"plugin", "provider"} and current_token:
current_lower = current_token.lower()
filtered = [c for c in choices if current_lower in c.lower()]
if filtered:
@@ -1996,188 +1992,13 @@ Come to love it when others take what you share, as there is no greater joy
) if isinstance(config,
dict) else None
if isinstance(provider_cfg, dict) and provider_cfg:
from Provider.metadata_provider import list_metadata_providers
from ProviderCore.registry import (
list_file_providers,
list_providers,
list_search_providers,
)
provider_availability = list_providers(config) or {}
search_availability = list_search_providers(config) or {}
file_availability = list_file_providers(config) or {}
meta_availability = list_metadata_providers(config) or {}
already_checked = {"matrix"}
for provider_name in provider_cfg.keys():
prov = str(provider_name or "").strip().lower()
if not prov or prov in already_checked:
continue
display = _provider_display_name(prov)
if prov == "alldebrid":
try:
from Provider.alldebrid import _get_debrid_api_key
from API.alldebrid import AllDebridClient
api_key = _get_debrid_api_key(config)
if not api_key:
_add_startup_check(
"DISABLED",
display,
provider=prov,
detail="Not configured"
)
else:
client = AllDebridClient(api_key)
base_url = str(
getattr(client,
"base_url",
"") or ""
).strip()
_add_startup_check(
"ENABLED",
display,
provider=prov,
detail=base_url or "Connected",
)
except Exception as exc:
_add_startup_check(
"DISABLED",
display,
provider=prov,
detail=str(exc)
)
continue
is_known = False
ok_val: Optional[bool] = None
if prov in provider_availability:
is_known = True
ok_val = bool(provider_availability.get(prov))
elif prov in search_availability:
is_known = True
ok_val = bool(search_availability.get(prov))
elif prov in file_availability:
is_known = True
ok_val = bool(file_availability.get(prov))
elif prov in meta_availability:
is_known = True
ok_val = bool(meta_availability.get(prov))
if not is_known:
_add_startup_check(
"UNKNOWN",
display,
provider=prov,
detail="Not registered"
)
else:
detail = "Configured" if ok_val else "Not configured"
ping_targets = _default_provider_ping_targets(prov)
if ping_targets:
ping_ok, ping_detail = _ping_first(ping_targets)
if ok_val:
detail = ping_detail
else:
detail = (
(detail + " | " +
ping_detail) if ping_detail else detail
)
_add_startup_check(
"ENABLED" if ok_val else "DISABLED",
display,
provider=prov,
detail=detail,
)
already_checked.add(prov)
default_search_providers = [
"openlibrary",
"libgen",
"youtube",
"bandcamp"
]
for prov in default_search_providers:
if prov in already_checked:
continue
display = _provider_display_name(prov)
ok_val = (
bool(search_availability.get(prov))
if prov in search_availability else False
)
ping_targets = _default_provider_ping_targets(prov)
ping_ok, ping_detail = (
_ping_first(ping_targets) if ping_targets else (False, "No ping target")
)
detail = ping_detail or (
"Available" if ok_val else "Unavailable"
)
if not ok_val:
detail = "Unavailable" + (
f" | {ping_detail}" if ping_detail else ""
)
for check in _collect_plugin_startup_checks(config):
_add_startup_check(
"ENABLED" if (ok_val and ping_ok) else "DISABLED",
display,
provider=prov,
detail=detail,
)
already_checked.add(prov)
if "0x0" not in already_checked:
ok_val = (
bool(file_availability.get("0x0"))
if "0x0" in file_availability else False
)
ping_ok, ping_detail = _ping_url("https://0x0.st")
detail = ping_detail
if not ok_val:
detail = "Unavailable" + (
f" | {ping_detail}" if ping_detail else ""
)
_add_startup_check(
"ENABLED" if (ok_val and ping_ok) else "DISABLED",
"0x0",
provider="0x0",
detail=detail,
)
if _has_provider(config, "matrix"):
try:
from Provider.matrix import Matrix
provider = Matrix(config)
matrix_conf = (
config.get("provider",
{}).get("matrix",
{}) if isinstance(config,
dict) else {}
)
homeserver = str(matrix_conf.get("homeserver") or "").strip()
room_id = str(matrix_conf.get("room_id") or "").strip()
if homeserver and not homeserver.startswith("http"):
homeserver = f"https://{homeserver}"
target = homeserver.rstrip("/")
if room_id:
target = (
target + (" " if target else "")
) + f"room:{room_id}"
_add_startup_check(
"ENABLED" if provider.validate() else "DISABLED",
"Matrix",
provider="matrix",
detail=target or
("Connected" if provider.validate() else "Not configured"),
)
except Exception as exc:
_add_startup_check(
"DISABLED",
"Matrix",
provider="matrix",
detail=str(exc)
str(check.get("status") or "UNKNOWN"),
str(check.get("name") or "Plugin"),
provider=str(check.get("plugin") or ""),
detail=str(check.get("detail") or ""),
files=check.get("files"),
)
if _has_store_subtype(config, "debrid"):
@@ -2213,9 +2034,7 @@ Come to love it when others take what you share, as there is no greater joy
)
try:
from tool.ytdlp import YtDlpTool
cookiefile = YtDlpTool(config).resolve_cookiefile()
cookiefile = resolve_cookies_path(config)
if cookiefile is not None:
_add_startup_check("FOUND", "Cookies", detail=str(cookiefile))
else:
MPV/pipeline_helper.py
+35 -25
View File
@@ -67,6 +67,7 @@ from SYS.config import load_config, reload_config # noqa: E402
from SYS.logger import set_debug, debug, set_thread_stream # noqa: E402
from SYS.repl_queue import enqueue_repl_command # noqa: E402
from SYS.utils import format_bytes # noqa: E402
from ProviderCore.registry import get_plugin, get_plugin_class # noqa: E402
REQUEST_PROP = "user-data/medeia-pipeline-request"
RESPONSE_PROP = "user-data/medeia-pipeline-response"
@@ -936,39 +937,36 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]:
"table": None,
}
cfg = load_config() or {}
plugin = get_plugin("ytdlp", cfg)
if plugin is None or not hasattr(plugin, "list_url_formats"):
return {
"success": False,
"stdout": "",
"stderr": "",
"error": "yt-dlp plugin unavailable",
"table": None,
}
try:
from tool.ytdlp import list_formats, is_browseable_format # noqa: WPS433
formats = plugin.list_url_formats(
url,
no_playlist=True,
timeout_seconds=25,
)
except Exception as exc:
return {
"success": False,
"stdout": "",
"stderr": "",
"error": f"yt-dlp tool unavailable: {type(exc).__name__}: {exc}",
"error": f"yt-dlp plugin probe failed: {type(exc).__name__}: {exc}",
"table": None,
}
cookiefile = None
try:
from tool.ytdlp import YtDlpTool # noqa: WPS433
cfg = load_config() or {}
cookie_path = YtDlpTool(cfg).resolve_cookiefile()
if cookie_path is not None:
cookiefile = str(cookie_path)
except Exception:
cookiefile = None
def _format_bytes(n: Any) -> str:
"""Format bytes using centralized utility."""
return format_bytes(n)
formats = list_formats(
url,
no_playlist=True,
cookiefile=cookiefile,
timeout_seconds=25,
)
if formats is None:
return {
"success": False,
@@ -990,9 +988,10 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]:
},
}
browseable = [f for f in formats if is_browseable_format(f)]
if browseable:
formats = browseable
try:
formats = plugin.filter_picker_formats(formats)
except Exception:
pass
# Debug: dump a short summary of the format list to the helper log.
try:
@@ -2040,8 +2039,19 @@ def main(argv: Optional[list[str]] = None) -> int:
# Publish yt-dlp supported domains for Lua menu filtering
try:
from tool.ytdlp import _build_supported_domains
domains = sorted(list(_build_supported_domains()))
plugin_class = get_plugin_class("ytdlp")
domains = []
if plugin_class is not None:
domains = sorted(
{
str(value).strip().lower()
for value in plugin_class.url_patterns()
if isinstance(value, str)
and str(value).strip()
and "://" not in str(value)
and not str(value).strip().endswith(":")
}
)
if domains:
# We join them into a space-separated string for Lua to parse easily
domains_str = " ".join(domains)
Provider/HIFI.py
+54 -2
View File
@@ -64,8 +64,7 @@ def _format_total_seconds(seconds: Any) -> str:
class HIFI(Provider):
PROVIDER_NAME = "hifi"
PLUGIN_NAME = "hifi"
TABLE_AUTO_STAGES = {
"hifi.track": ["download-file"],
@@ -2092,3 +2091,56 @@ class HIFI(Provider):
pass
return True
def expand_selection(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
table_type: str = "",
**_kwargs: Any,
) -> Optional[List[Any]]:
_ = ctx
if stage_is_last:
return None
normalized_table = str(table_type or "").strip().lower()
if normalized_table != "hifi.album":
return None
try:
contexts = self._extract_album_selection_context(selected_items)
except Exception:
return None
if not contexts:
return None
track_items: List[Any] = []
seen_track_ids: set[int] = set()
for album_id, album_title, artist_name in contexts:
try:
track_results = self._tracks_for_album(
album_id=album_id,
album_title=album_title,
artist_name=artist_name,
limit=500,
)
except Exception:
track_results = []
for track in track_results or []:
try:
metadata = getattr(track, "full_metadata", None)
track_id = None
if isinstance(metadata, dict):
raw_id = metadata.get("trackId") or metadata.get("id")
track_id = int(raw_id) if raw_id is not None else None
if track_id is not None:
if track_id in seen_track_ids:
continue
seen_track_ids.add(track_id)
except Exception:
pass
track_items.append(track)
return track_items or None
Provider/Tidal.py
+58 -2
View File
@@ -64,7 +64,7 @@ def _format_total_seconds(seconds: Any) -> str:
class Tidal(Provider):
PROVIDER_NAME = "tidal"
PLUGIN_NAME = "tidal"
TABLE_AUTO_STAGES = {
"tidal.track": ["download-file"],
@@ -82,7 +82,7 @@ class Tidal(Provider):
"tidal.com",
"listen.tidal.com",
)
URL = URL_DOMAINS
URL = URL_DOMAINS + ("tidal:",)
"""Provider that targets the Tidal search endpoint.
The CLI can supply a list of fail-over URLs via ``provider.tidal.api_urls`` or
@@ -210,6 +210,9 @@ class Tidal(Provider):
self.api_timeout = 10.0
self.api_clients = [TidalApiClient(base_url=url, timeout=self.api_timeout) for url in self.api_urls]
def resolve_playback_path(self, item: Any, **_kwargs: Any) -> Optional[str]:
return resolve_tidal_manifest_path(item)
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
"""Parse inline `key:value` query arguments.
@@ -2399,3 +2402,56 @@ class Tidal(Provider):
pass
return True
def expand_selection(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
table_type: str = "",
**_kwargs: Any,
) -> Optional[List[Any]]:
_ = ctx
if stage_is_last:
return None
normalized_table = str(table_type or "").strip().lower()
if normalized_table != "tidal.album":
return None
try:
contexts = self._extract_album_selection_context(selected_items)
except Exception:
return None
if not contexts:
return None
track_items: List[Any] = []
seen_track_ids: set[int] = set()
for album_id, album_title, artist_name in contexts:
try:
track_results = self._tracks_for_album(
album_id=album_id,
album_title=album_title,
artist_name=artist_name,
limit=500,
)
except Exception:
track_results = []
for track in track_results or []:
try:
metadata = getattr(track, "full_metadata", None)
track_id = None
if isinstance(metadata, dict):
raw_id = metadata.get("trackId") or metadata.get("id")
track_id = int(raw_id) if raw_id is not None else None
if track_id is not None:
if track_id in seen_track_ids:
continue
seen_track_ids.add(track_id)
except Exception:
pass
track_items.append(track)
return track_items or None
Provider/__init__.py
+3 -3
View File
@@ -1,7 +1,7 @@
"""Provider plugin modules.
"""Built-in plugin modules.
Concrete provider implementations live in this package.
The public entrypoint/registry is ProviderCore.registry.
Concrete built-in plugins live in this package.
The public registry lives in ProviderCore.registry.
"""
# Register providers with the strict ResultTable adapter system
Provider/alldebrid.py
+72 -19
View File
@@ -351,7 +351,7 @@ def _dispatch_alldebrid_magnet_search(
if callable(exec_fn):
exec_fn(
None,
["-provider", "alldebrid", f"ID={magnet_id}"],
["-plugin", "alldebrid", f"ID={magnet_id}"],
config,
)
except Exception:
@@ -493,7 +493,7 @@ def download_magnet(
def expand_folder_item(
item: Any,
get_search_provider: Optional[Callable[[str, Dict[str, Any]], Any]],
get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
config: Dict[str, Any],
) -> Tuple[List[Any], Optional[str]]:
table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table")
@@ -517,15 +517,15 @@ def expand_folder_item(
except Exception:
magnet_id = None
if magnet_id is None or get_search_provider is None:
if magnet_id is None or get_search_plugin is None:
return [], None
provider = get_search_provider("alldebrid", config) if get_search_provider else None
if provider is None:
plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None
if plugin is None:
return [], None
try:
files = provider.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)})
files = plugin.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)})
except Exception:
files = []
@@ -609,7 +609,7 @@ class AllDebrid(TableProviderMixin, Provider):
- Drill-down: Selecting a folder row (@N) fetches and displays all files
SELECTION FLOW:
1. User runs: search-file -provider alldebrid "ubuntu"
1. User runs: search-file -plugin alldebrid "ubuntu"
2. Results show magnet folders and (optionally) files
3. User selects a row: @1
4. Selection metadata routes to download-file with -url alldebrid:magnet:<id>
@@ -619,7 +619,7 @@ class AllDebrid(TableProviderMixin, Provider):
# Magnet URIs should be routed through this provider.
TABLE_AUTO_STAGES = {"alldebrid": ["download-file"]}
AUTO_STAGE_USE_SELECTION_ARGS = True
URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲")
URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲", "alldebrid.com")
URL_DOMAINS = ()
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
@@ -949,12 +949,10 @@ class AllDebrid(TableProviderMixin, Provider):
except Exception:
return None
@classmethod
def download_for_pipe_result(
cls,
def resolve_pipe_result_download(
self,
result: Any,
pipe_obj: Optional[PipeObject],
config: Dict[str, Any],
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
"""Download a remote provider result on behalf of add-file."""
@@ -1026,8 +1024,7 @@ class AllDebrid(TableProviderMixin, Provider):
download_dir = Path(tempfile.mkdtemp(prefix="add-file-alldebrid-"))
try:
provider = cls(config)
downloaded_path = provider.download(search_result, download_dir)
downloaded_path = self.download(search_result, download_dir)
if not downloaded_path:
shutil.rmtree(download_dir, ignore_errors=True)
return None, None, None
@@ -1049,6 +1046,62 @@ class AllDebrid(TableProviderMixin, Provider):
log(f"[alldebrid] add-file download failed: {exc}", file=sys.stderr)
shutil.rmtree(download_dir, ignore_errors=True)
return None, None, None
def status_summary(self) -> Dict[str, Any]:
try:
api_key = _get_debrid_api_key(self.config)
if not api_key:
return {
"status": "DISABLED",
"name": self.label,
"plugin": self.name,
"detail": "Not configured",
}
client = AllDebridClient(api_key)
base_url = str(getattr(client, "base_url", "") or "").strip()
return {
"status": "ENABLED",
"name": self.label,
"plugin": self.name,
"detail": base_url or "Connected",
}
except Exception as exc:
return {
"status": "DISABLED",
"name": self.label,
"plugin": self.name,
"detail": str(exc),
}
def resolve_url(self, url: str, **_kwargs: Any) -> str:
target = str(url or "").strip()
if not target.startswith(("http://", "https://")):
return target
try:
parsed = urlparse(target)
host = str(parsed.netloc or "").lower()
path = str(parsed.path or "")
except Exception:
return target
if host != "alldebrid.com" or not path.startswith("/f/"):
return target
api_key = _get_debrid_api_key(self.config)
if not api_key:
return target
try:
client = AllDebridClient(str(api_key))
unlocked = client.unlock_link(target)
if isinstance(unlocked, str) and unlocked.strip():
return unlocked.strip()
except Exception:
pass
return target
def download_items(
self,
result: SearchResult,
@@ -1413,7 +1466,7 @@ class AllDebrid(TableProviderMixin, Provider):
"provider_view": "files",
# Selection metadata for table system
"_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
"_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
"_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
}
results.append(
@@ -1528,7 +1581,7 @@ class AllDebrid(TableProviderMixin, Provider):
"magnet_name": magnet_name,
# Selection metadata: allow @N expansion to drive downloads directly
"_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
"_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
"_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
},
)
)
@@ -1629,7 +1682,7 @@ class AllDebrid(TableProviderMixin, Provider):
table.set_table_metadata({"provider": "alldebrid", "view": "files", "magnet_id": magnet_id})
except Exception:
pass
table.set_source_command("download-file", ["-provider", "alldebrid"])
table.set_source_command("download-file", ["-plugin", "alldebrid"])
results_payload: List[Dict[str, Any]] = []
for r in files or []:
@@ -1662,7 +1715,7 @@ class AllDebrid(TableProviderMixin, Provider):
try:
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column
def _as_payload(item: Any) -> Dict[str, Any]:
@@ -1853,7 +1906,7 @@ try:
return ["-title", row.title or ""]
register_provider(
register_plugin(
"alldebrid",
_adapter,
columns=_columns_factory,
Provider/example_provider.py
+9 -9
View File
@@ -1,4 +1,4 @@
"""Example provider that uses the new `ResultTable` API.
"""Example plugin that uses the new `ResultTable` API.
This module demonstrates a minimal provider adapter that yields `ResultModel`
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
@@ -8,7 +8,7 @@ Run this to see sample output:
python -m Provider.example_provider
Example usage (piped selector):
provider-table -provider example -sample | select -select 1 | add-file -store default
plugin-table -plugin example -sample | select -select 1 | add-file -store default
"""
from __future__ import annotations
@@ -105,9 +105,9 @@ def selection_fn(row: ResultModel) -> List[str]:
return ["-title", row.title]
# Register the provider with the registry so callers can discover it by name
from SYS.result_table_adapters import register_provider
register_provider(
# Register the plugin with the registry so callers can discover it by name
from SYS.result_table_adapters import register_plugin
register_plugin(
"example",
adapter,
columns=columns_factory,
@@ -223,17 +223,17 @@ def demo() -> None:
def demo_with_selection(idx: int = 0) -> None:
"""Demonstrate how a cmdlet would use provider registration and selection args.
"""Demonstrate how a cmdlet would use plugin registration and selection args.
- Fetch the registered provider by name
- Fetch the registered plugin by name
- Build rows via adapter
- Render the table
- Show the selection args for the chosen row; these are the args a cmdlet
would append when the user picks that row.
"""
from SYS.result_table_adapters import get_provider
from SYS.result_table_adapters import get_plugin
provider = get_provider("example")
provider = get_plugin("example")
rows = list(provider.adapter(SAMPLE_ITEMS))
cols = provider.get_columns(rows)
Provider/fileio.py
+1 -1
View File
@@ -50,7 +50,7 @@ def _extract_key(payload: Any) -> Optional[str]:
class FileIO(Provider):
"""File provider for file.io."""
PROVIDER_NAME = "file.io"
PLUGIN_NAME = "file.io"
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
Provider/hello_provider.py
+5 -4
View File
@@ -1,6 +1,6 @@
"""Example provider template for use as a starter kit.
"""Example plugin template for use as a starter kit.
This minimal provider demonstrates the typical hooks a provider may implement:
This minimal plugin demonstrates the typical hooks a plugin may implement:
- `validate()` to assert it's usable
- `search()` to return `SearchResult` items
- `download()` to persist a sample file (useful for local tests)
@@ -17,13 +17,14 @@ from ProviderCore.base import Provider, SearchResult
class HelloProvider(Provider):
"""Very small example provider suitable as a template.
"""Very small example plugin suitable as a template.
- Table name: `hello`
- Usage: `search-file -provider hello "query"`
- Usage: `search-file -plugin hello "query"`
- Selecting a row and piping into `download-file` will call `download()`.
"""
PLUGIN_NAME = "hello"
URL = ("hello:",)
URL_DOMAINS = ()
Provider/internetarchive.py
+2 -2
View File
@@ -594,9 +594,9 @@ class InternetArchive(Provider):
"""Internet Archive provider using the `internetarchive` Python module.
Supports:
- search-file -provider internetarchive <query>
- search-file -plugin internetarchive <query>
- download-file / provider.download() from search results
- add-file -provider internetarchive (uploads)
- add-file -plugin internetarchive (uploads)
"""
URL = ("archive.org",)
Provider/matrix.py
+18 -3
View File
@@ -294,7 +294,7 @@ class Matrix(TableProviderMixin, Provider):
- MIME detection: Automatic content type classification for Matrix msgtype
SELECTION FLOW:
1. User runs: search-file -provider matrix "room" (or .matrix -list-rooms)
1. User runs: search-file -plugin matrix "room" (or .matrix -list-rooms)
2. Results show available joined rooms
3. User selects rooms: @1 @2 (or @1,2)
4. Selection triggers upload of pending files to selected rooms
@@ -368,6 +368,21 @@ class Matrix(TableProviderMixin, Provider):
and matrix_conf.get("access_token")
)
def status_summary(self) -> Dict[str, Any]:
matrix_conf = self.config.get("provider", {}).get("matrix", {}) if isinstance(self.config, dict) else {}
homeserver = str(matrix_conf.get("homeserver") or "").strip()
room_id = str(matrix_conf.get("room_id") or "").strip()
detail = homeserver
if room_id:
detail = (detail + (" " if detail else "")) + f"room:{room_id}"
enabled = bool(self.validate())
return {
"status": "ENABLED" if enabled else "DISABLED",
"name": self.label,
"plugin": self.name,
"detail": detail or ("Connected" if enabled else "Not configured"),
}
def search(
self,
query: str,
@@ -767,7 +782,7 @@ class Matrix(TableProviderMixin, Provider):
# Minimal provider registration for the new table system
try:
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
def _convert_search_result_to_model(sr: Any) -> ResultModel:
@@ -850,7 +865,7 @@ try:
return ["-title", row.title or ""]
register_provider(
register_plugin(
"matrix",
_adapter,
columns=_columns_factory,
Provider/metadata_provider.py
+355 -2
View File
@@ -40,6 +40,42 @@ except ImportError: # pragma: no cover - optional
yt_dlp = None
def _dedup_text_values(values: List[str]) -> List[str]:
out: List[str] = []
seen: set[str] = set()
for value in values or []:
if value is None:
continue
text = str(value).strip()
if not text:
continue
key = text.lower()
if key in seen:
continue
seen.add(key)
out.append(text)
return out
def _filter_default_scraped_tags(tags: List[str]) -> List[str]:
blocked = {"title", "artist", "source"}
out: List[str] = []
seen: set[str] = set()
for tag in tags or []:
text = str(tag or "").strip()
if not text:
continue
namespace = text.split(":", 1)[0].strip().lower() if ":" in text else ""
if namespace in blocked:
continue
key = text.lower()
if key in seen:
continue
seen.add(key)
out.append(text)
return out
class MetadataProvider(ABC):
"""Base class for metadata providers (music, movies, books, etc.)."""
@@ -122,6 +158,64 @@ class MetadataProvider(ABC):
return False
def default_subject_scrape_priority(self) -> int:
"""Priority used when `get-tag -scrape` is invoked without an explicit provider."""
return 0
def url_scrape_priority(self, url: str) -> int:
"""Priority for handling a raw URL passed to `get-tag -scrape <url>`."""
_ = url
return 0
def resolve_subject_query(
self,
result: Any,
get_field: Any,
*,
backend: Any = None,
file_hash: Optional[str] = None,
) -> Optional[str]:
"""Resolve a provider-specific query from the current subject/result."""
_ = backend
_ = file_hash
return self.extract_url_query(result, get_field)
def prefers_store_tag_overwrite(self) -> bool:
"""Whether direct subject scrapes should replace the store tag set."""
return False
def filter_tags_for_selection(self, tags: List[str]) -> List[str]:
"""Filter scraped tags before presenting a selectable metadata row."""
return _filter_default_scraped_tags(tags)
def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]:
"""Filter scraped tags before applying them to an existing store-backed item."""
return self.filter_tags_for_selection(tags)
def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]:
"""Return a URL scrape payload for `get-tag -scrape <url>` when supported."""
items = self.search(url, limit=1)
if not items:
return None
item = items[0] if isinstance(items[0], dict) else {}
try:
tags = [str(t) for t in self.to_tags(item) if t is not None]
except Exception:
tags = []
return {
"title": item.get("title"),
"tag": _dedup_text_values(tags),
"formats": [],
"playlist_items": [],
}
class ITunesProvider(MetadataProvider):
"""Metadata provider using the iTunes Search API."""
@@ -1015,6 +1109,226 @@ class YtdlpMetadataProvider(MetadataProvider):
def emits_direct_tags(self) -> bool:
return True
def default_subject_scrape_priority(self) -> int:
return 100
def url_scrape_priority(self, url: str) -> int:
text = str(url or "").strip()
if not text.startswith(("http://", "https://")):
return 0
return 100
def prefers_store_tag_overwrite(self) -> bool:
return True
def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]:
return _dedup_text_values(tags)
def _resolve_candidate_urls_for_subject(
self,
result: Any,
get_field: Any,
*,
backend: Any = None,
file_hash: Optional[str] = None,
) -> List[str]:
try:
from SYS.metadata import normalize_urls
except Exception:
normalize_urls = None # type: ignore[assignment]
urls: List[str] = []
if backend is not None and file_hash:
try:
backend_urls = backend.get_url(file_hash, config=self.config)
if backend_urls:
if normalize_urls:
urls.extend(normalize_urls(backend_urls))
else:
urls.extend(
[str(u).strip() for u in backend_urls if isinstance(u, str) and str(u).strip()]
)
except Exception:
pass
try:
meta = backend.get_metadata(file_hash, config=self.config)
if isinstance(meta, dict) and meta.get("url"):
raw = meta.get("url")
if normalize_urls:
urls.extend(normalize_urls(raw))
elif isinstance(raw, list):
urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()])
elif isinstance(raw, str) and raw.strip():
urls.append(raw.strip())
except Exception:
pass
for key in ("url", "webpage_url", "source_url", "target"):
val = get_field(result, key, None)
if not val:
continue
if normalize_urls:
urls.extend(normalize_urls(val))
continue
if isinstance(val, str) and val.strip():
urls.append(val.strip())
elif isinstance(val, list):
urls.extend([str(u).strip() for u in val if isinstance(u, str) and str(u).strip()])
meta_field = get_field(result, "metadata", None)
if isinstance(meta_field, dict) and meta_field.get("url"):
raw = meta_field.get("url")
if normalize_urls:
urls.extend(normalize_urls(raw))
elif isinstance(raw, list):
urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()])
elif isinstance(raw, str) and raw.strip():
urls.append(raw.strip())
return _dedup_text_values(urls)
def _pick_supported_subject_url(self, urls: List[str]) -> Optional[str]:
if not urls:
return None
def _is_hydrus_file_url(u: str) -> bool:
text = str(u or "").strip().lower()
return bool(text and "/get_files/file" in text and "hash=" in text)
candidates = []
for url in urls:
text = str(url or "").strip()
if not text.startswith(("http://", "https://")):
continue
if _is_hydrus_file_url(text):
continue
candidates.append(text)
if not candidates:
return None
try:
from tool.ytdlp import is_url_supported_by_ytdlp
for text in candidates:
try:
if is_url_supported_by_ytdlp(text):
return text
except Exception:
continue
except Exception:
pass
return candidates[0] if candidates else None
def resolve_subject_query(
self,
result: Any,
get_field: Any,
*,
backend: Any = None,
file_hash: Optional[str] = None,
) -> Optional[str]:
candidate_urls = self._resolve_candidate_urls_for_subject(
result,
get_field,
backend=backend,
file_hash=file_hash,
)
return self._pick_supported_subject_url(candidate_urls)
@staticmethod
def _extract_url_formats(formats: Any) -> List[tuple[str, str]]:
if not isinstance(formats, list):
return []
video_formats: Dict[str, Dict[str, Any]] = {}
audio_formats: Dict[str, Dict[str, Any]] = {}
for fmt in formats:
if not isinstance(fmt, dict):
continue
vcodec = fmt.get("vcodec", "none")
acodec = fmt.get("acodec", "none")
height = fmt.get("height")
ext = fmt.get("ext", "unknown")
format_id = fmt.get("format_id", "")
tbr = fmt.get("tbr", 0)
abr = fmt.get("abr", 0)
if vcodec and vcodec != "none" and height:
if int(height) < 480:
continue
res_key = f"{int(height)}p"
if res_key not in video_formats or tbr > video_formats[res_key].get("tbr", 0):
video_formats[res_key] = {
"label": f"{int(height)}p ({ext})",
"format_id": str(format_id),
"tbr": tbr,
}
elif acodec and acodec != "none" and (not vcodec or vcodec == "none"):
audio_key = f"audio_{abr}"
if audio_key not in audio_formats or abr > audio_formats[audio_key].get("abr", 0):
audio_formats[audio_key] = {
"label": f"audio ({ext})",
"format_id": str(format_id),
"abr": abr,
}
result: List[tuple[str, str]] = []
for res in sorted(video_formats.keys(), key=lambda value: int(value.replace("p", "")), reverse=True):
fmt = video_formats[res]
result.append((str(fmt.get("label") or res), str(fmt.get("format_id") or "")))
if audio_formats:
best_audio_key = max(audio_formats.keys(), key=lambda key: float(audio_formats[key].get("abr", 0) or 0))
fmt = audio_formats[best_audio_key]
result.append((str(fmt.get("label") or "audio"), str(fmt.get("format_id") or "")))
return [entry for entry in result if entry[1]]
@staticmethod
def _build_playlist_items(raw: Dict[str, Any]) -> List[Dict[str, Any]]:
entries = raw.get("entries")
if not isinstance(entries, list):
return []
playlist_items: List[Dict[str, Any]] = []
for idx, entry in enumerate(entries, 1):
if not isinstance(entry, dict):
continue
playlist_items.append(
{
"index": idx,
"id": entry.get("id", f"track_{idx}"),
"title": entry.get("title", entry.get("id", f"Track {idx}")),
"duration": entry.get("duration", 0),
"url": entry.get("url") or entry.get("webpage_url", ""),
}
)
return playlist_items
def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]:
info = self._extract_info(url)
if not isinstance(info, dict):
return None
item = {
"title": info.get("title") or "",
"artist": str(info.get("artist") or info.get("uploader") or info.get("channel") or ""),
"album": str(info.get("album") or info.get("playlist_title") or ""),
"year": str((str(info.get("release_date") or "") or str(info.get("upload_date") or ""))[:4]),
"provider": self.name,
"url": str(url or "").strip(),
"raw": info,
}
tags = _dedup_text_values([str(tag) for tag in self.to_tags(item) if tag is not None])
return {
"title": item.get("title") or None,
"tag": tags,
"formats": self._extract_url_formats(info.get("formats", [])),
"playlist_items": self._build_playlist_items(info),
}
def _coerce_archive_field_list(value: Any) -> List[str]:
"""Coerce an Archive.org metadata field to a list of strings."""
@@ -1420,7 +1734,7 @@ try:
from typing import Iterable
from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
def _ensure_search_result(item: Any) -> SearchResult:
if isinstance(item, SearchResult):
@@ -1526,7 +1840,7 @@ try:
return ["-url", url]
return ["-title", row.title or ""]
register_provider(
register_plugin(
"openlibrary",
_adapter,
columns=_columns_factory,
@@ -1671,3 +1985,42 @@ def get_metadata_provider(name: str,
except Exception as exc:
log(f"Provider init failed for '{name}': {exc}", file=sys.stderr)
return None
def get_default_subject_scrape_provider(
config: Optional[Dict[str, Any]] = None,
) -> Optional[MetadataProvider]:
best_provider: Optional[MetadataProvider] = None
best_priority = 0
for cls in _METADATA_PROVIDERS.values():
try:
provider = cls(config)
priority = int(provider.default_subject_scrape_priority())
except Exception:
continue
if priority > best_priority:
best_priority = priority
best_provider = provider
return best_provider
def get_metadata_provider_for_url(
url: str,
config: Optional[Dict[str, Any]] = None,
) -> Optional[MetadataProvider]:
text = str(url or "").strip()
if not text:
return None
best_provider: Optional[MetadataProvider] = None
best_priority = 0
for cls in _METADATA_PROVIDERS.values():
try:
provider = cls(config)
priority = int(provider.url_scrape_priority(text))
except Exception:
continue
if priority > best_priority:
best_priority = priority
best_provider = provider
return best_provider
Provider/soulseek.py
+2 -2
View File
@@ -216,7 +216,7 @@ def _suppress_aioslsk_noise() -> Any:
class Soulseek(Provider):
TABLE_AUTO_STAGES = {
"soulseek": ["download-file", "-provider", "soulseek"],
"soulseek": ["download-file", "-plugin", "soulseek"],
}
"""Search provider for Soulseek P2P network."""
@@ -623,7 +623,7 @@ class Soulseek(Provider):
media_kind="audio",
size_bytes=item["size"],
columns=columns,
selection_action=["download-file", "-provider", "soulseek"],
selection_action=["download-file", "-plugin", "soulseek"],
full_metadata={
"username": item["username"],
"filename": item["filename"],
Provider/vimm.py
+10 -10
View File
@@ -37,15 +37,15 @@ class Vimm(TableProviderMixin, Provider):
2) Each row carries explicit selection args: `['-url', '<full-url>']`.
Using an explicit `-url` flag avoids ambiguity during argument
parsing (some cmdlets accept positional URLs, others accept flags).
3) The CLI's expansion logic places selection args *before* provider
source args (e.g., `-provider vimm`) so the first positional token is
the intended URL (not an unknown flag like `-provider`).
3) The CLI's expansion logic places selection args *before* plugin
source args (e.g., `-plugin vimm`) so the first positional token is
the intended URL (not an unknown flag like `-plugin`).
- Why this approach? Argument parsing treats the *first* unrecognized token
as a positional value (commonly interpreted as a URL). If a provider
injects hints like `-provider vimm` *before* a bare URL, the parser can
misinterpret `-provider` as the URL, causing confusing attempts to
download `-provider`. By using `-url` and ensuring the URL appears first
as a positional value (commonly interpreted as a URL). If a plugin
injects hints like `-plugin vimm` *before* a bare URL, the parser can
misinterpret `-plugin` as the URL, causing confusing attempts to
download `-plugin`. By using `-url` and ensuring the URL appears first
we avoid that class of bugs and make `@N` -> `download-file`/`add-file`
flows reliable.
@@ -56,7 +56,7 @@ class Vimm(TableProviderMixin, Provider):
URL_DOMAINS = ("vimm.net",)
def get_source_command(self, args_list: List[str]) -> Tuple[str, List[str]]:
return "search-file", ["-provider", self.name]
return "search-file", ["-plugin", self.name]
REGION_CHOICES = [
{"value": "1", "text": "Argentina"},
@@ -807,7 +807,7 @@ class Vimm(TableProviderMixin, Provider):
# Minimal provider registration
try:
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
from SYS.result_table_api import ResultModel, title_column, metadata_column
def _convert_search_result_to_model(sr):
@@ -857,7 +857,7 @@ try:
return ["-title", row.title or ""]
register_provider(
register_plugin(
"vimm",
_adapter,
columns=_columns_factory,
Provider/youtube.py
+3 -3
View File
@@ -21,7 +21,7 @@ class YouTube(TableProviderMixin, Provider):
- _selection_args: For @N expansion control and download-file routing
SELECTION FLOW:
1. User runs: search-file -provider youtube "linux tutorial"
1. User runs: search-file -plugin youtube "linux tutorial"
2. Results show video rows with uploader, duration, views
3. User selects a video: @1
4. Selection metadata routes to download-file with the YouTube URL
@@ -121,7 +121,7 @@ class YouTube(TableProviderMixin, Provider):
# Minimal provider registration for the new table system
try:
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
def _convert_search_result_to_model(sr: Any) -> ResultModel:
@@ -206,7 +206,7 @@ try:
return ["-title", row.title or ""]
register_provider(
register_plugin(
"youtube",
_adapter,
columns=_columns_factory,
Provider/ytdlp.py
+1204 -150
View File
File diff suppressed because it is too large Load Diff
Provider/zeroxzero.py
+2 -2
View File
@@ -11,8 +11,8 @@ from SYS.logger import log
class ZeroXZero(Provider):
"""File provider for 0x0.st."""
NAME = "0x0"
PROVIDER_ALIASES = ("zeroxzero",)
PLUGIN_NAME = "0x0"
PLUGIN_ALIASES = ("zeroxzero",)
def upload(self, file_path: str, **kwargs: Any) -> str:
from API.HTTP import HTTPClient
ProviderCore/__init__.py
+4 -3
View File
@@ -1,5 +1,6 @@
"""Provider core modules.
"""Plugin core modules.
This package contains the provider framework (base types, registry, and shared helpers).
Concrete provider implementations live in the `Provider/` package.
This package contains the plugin framework (base types, registry, and shared
helpers). Built-in plugins continue to live in the `Provider/` package for
backward compatibility.
"""
ProviderCore/base.py
+139 -16
View File
@@ -10,9 +10,9 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple, Callable
@dataclass
class SearchResult:
"""Unified search result format across all search providers."""
"""Unified search result format across all search plugins."""
table: str # Provider name: "libgen", "soulseek", "bandcamp", "youtube", etc.
table: str # Plugin name: "libgen", "soulseek", "bandcamp", "youtube", etc.
title: str # Display title/filename
path: str # Download target (URL, path, magnet, identifier)
@@ -84,7 +84,7 @@ class SearchResult:
def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]:
"""Extract inline key:value arguments from a provider search query."""
"""Extract inline key:value arguments from a plugin search query."""
query_text = str(raw_query or "").strip()
if not query_text:
@@ -112,10 +112,10 @@ def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]:
class Provider(ABC):
"""Unified provider base class.
"""Unified plugin base class.
This replaces the older split between "search providers" and "file providers".
Concrete providers may implement any subset of:
This replaces the older split between search and upload providers.
Concrete plugins may implement any subset of:
- search(query, ...)
- download(result, output_dir)
- upload(file_path, ...)
@@ -124,7 +124,8 @@ class Provider(ABC):
"""
URL: Sequence[str] = ()
NAME: str = ""
PLUGIN_NAME: str = ""
PLUGIN_ALIASES: Sequence[str] = ()
# Optional provider-driven defaults for what to do when a user selects @N from a
# provider table. The CLI uses this to auto-insert stages (e.g. download-file)
@@ -141,24 +142,23 @@ class Provider(ABC):
# Used for dynamically generating config panels (e.g., missing credentials).
REQUIRED_CONFIG_KEYS: Sequence[str] = ()
# Some providers implement `upload()` but are not intended to be used as
# generic "file host" providers via `add-file -provider ...`.
# Some plugins implement `upload()` but are not intended to be used as
# generic "file host" plugins via `add-file -plugin ...`.
EXPOSE_AS_FILE_PROVIDER: bool = True
def __init__(self, config: Optional[Dict[str, Any]] = None):
self.config = config or {}
# Prioritize explicit NAME property for the instance name
self.name = str(
getattr(self, "NAME", None)
or getattr(self, "PROVIDER_NAME", None)
getattr(self, "PLUGIN_NAME", None)
or self.__class__.__name__
).lower()
@property
def label(self) -> str:
"""Friendly display name for the provider."""
if hasattr(self, "NAME") and self.NAME:
name = str(self.NAME)
"""Friendly display name for the plugin."""
name = str(getattr(self, "PLUGIN_NAME", None) or self.__class__.__name__)
if name:
if name.lower() == "loc":
return "LoC"
if name.lower() == "openlibrary":
@@ -186,7 +186,7 @@ class Provider(ABC):
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""Return metadata for the results table."""
return {"provider": self.name}
return {"plugin": self.name}
def get_source_command(self, args_list: List[str]) -> Tuple[str, List[str]]:
"""Return the command and arguments that produced this search result.
@@ -308,6 +308,49 @@ class Provider(ABC):
_ = config
return 0
def resolve_pipe_result_download(
self,
result: Any,
pipe_obj: Any,
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
"""Materialize a piped plugin result into a local file for add-file."""
_ = result
_ = pipe_obj
return None, None, None
def expand_selection(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
table_type: str = "",
**_kwargs: Any,
) -> Optional[List[Any]]:
"""Optionally expand a selection into downstream items for non-terminal pipelines."""
_ = selected_items
_ = ctx
_ = stage_is_last
_ = table_type
return None
def status_summary(self) -> Dict[str, Any]:
"""Return plugin-owned status details for startup/status views."""
enabled = False
try:
enabled = bool(self.validate())
except Exception:
enabled = False
return {
"status": "ENABLED" if enabled else "DISABLED",
"name": self.label,
"plugin": self.name,
"detail": "Configured" if enabled else "Not configured",
}
def handle_url(self, url: str, *, output_dir: Optional[Path] = None) -> Tuple[bool, Optional[Path | Dict[str, Any]]]:
"""Optional provider override to parse and act on URLs."""
@@ -315,6 +358,67 @@ class Provider(ABC):
_ = output_dir
return False, None
def download_url(self, url: str, output_dir: Path, **_kwargs: Any) -> Optional[Any]:
"""Optional direct-URL download hook used by generic cmdlets."""
_ = url
_ = output_dir
return None
def resolve_url(self, url: str, **_kwargs: Any) -> str:
"""Optionally normalize or exchange a URL before downstream use."""
return str(url or "")
def resolve_playback_path(self, item: Any, **_kwargs: Any) -> Optional[str]:
"""Optionally turn a plugin-owned item into a playable local path or URL."""
_ = item
return None
def list_url_formats(self, url: str, **_kwargs: Any) -> Optional[List[Dict[str, Any]]]:
"""Optionally return picker-friendly format metadata for a URL."""
_ = url
return None
def filter_picker_formats(
self,
formats: List[Dict[str, Any]],
**_kwargs: Any,
) -> List[Dict[str, Any]]:
"""Optionally filter or reorder raw format rows before UI display."""
return list(formats or [])
def enrich_playlist_entries(
self,
entries: List[Dict[str, Any]],
**_kwargs: Any,
) -> Optional[List[Dict[str, Any]]]:
"""Optionally expand lightweight playlist entries with richer metadata."""
_ = entries
return None
def maybe_show_picker(
self,
*,
url: str,
item: Optional[Any] = None,
parsed: Optional[Dict[str, Any]] = None,
config: Optional[Dict[str, Any]] = None,
quiet_mode: bool = False,
) -> Optional[int]:
"""Optional hook for plugins that want to render an interactive picker/table."""
_ = url
_ = item
_ = parsed
_ = config
_ = quiet_mode
return None
def upload(self, file_path: str, **kwargs: Any) -> str:
"""Upload a file and return a URL or identifier."""
raise NotImplementedError(f"Provider '{self.name}' does not support upload")
@@ -419,6 +523,25 @@ class Provider(ABC):
patterns.append(candidate)
return tuple(patterns)
@classmethod
def selection_url_prefixes(cls) -> Tuple[str, ...]:
"""Return URL-like prefixes that selection parsing should treat as URLs."""
prefixes: List[str] = []
seen: set[str] = set()
for pattern in cls.url_patterns():
try:
candidate = str(pattern or "").strip().lower()
except Exception:
continue
if not candidate:
continue
if "://" in candidate or candidate.endswith(":") or "🧲" in candidate:
if candidate not in seen:
seen.add(candidate)
prefixes.append(candidate)
return tuple(prefixes)
class SearchProvider(Provider):
"""Compatibility alias for older code.
ProviderCore/registry.py
+225 -92
View File
@@ -1,17 +1,21 @@
"""Provider registry.
"""Plugin registry.
Concrete provider implementations live in the ``Provider`` package. This module
is the single source of truth for discovery, metadata, and lifecycle helpers
for those plugins.
Built-in plugin implementations live in the ``Provider`` package. External user
plugins can be dropped into a repo-local ``plugins/`` directory or discovered
via environment-configured plugin paths.
"""
from __future__ import annotations
from functools import lru_cache
import hashlib
import importlib
import importlib.util
import os
import pkgutil
import sys
from dataclasses import dataclass, field
from pathlib import Path
from types import ModuleType
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Type
from urllib.parse import urlparse
@@ -21,21 +25,85 @@ from SYS.logger import log, debug
from ProviderCore.base import FileProvider, Provider, SearchProvider, SearchResult
def download_soulseek_file(*args: Any, **kwargs: Any) -> Any:
"""Lazy proxy for the soulseek downloader.
_EXTERNAL_PLUGIN_ENV_VARS: tuple[str, ...] = ("MM_PLUGIN_PATH", "MEDEIA_PLUGIN_PATH")
Importing the provider modules can be expensive; keeping this lazy avoids
paying that cost at registry import time.
"""
from Provider.soulseek import download_soulseek_file as _download
def _repo_root() -> Path:
try:
return Path(__file__).resolve().parents[1]
except Exception:
return Path.cwd()
return _download(*args, **kwargs)
def _iter_external_plugin_dirs() -> Tuple[Path, ...]:
seen: set[str] = set()
dirs: List[Path] = []
candidates: List[Path] = [_repo_root() / "plugins"]
try:
cwd_plugins = Path.cwd() / "plugins"
if cwd_plugins not in candidates:
candidates.append(cwd_plugins)
except Exception:
pass
for env_name in _EXTERNAL_PLUGIN_ENV_VARS:
raw_value = str(os.environ.get(env_name, "") or "").strip()
if not raw_value:
continue
for chunk in raw_value.split(os.pathsep):
text = str(chunk or "").strip().strip('"')
if not text:
continue
candidates.append(Path(text).expanduser())
for candidate in candidates:
try:
resolved = candidate.resolve()
except Exception:
resolved = candidate
key = str(resolved).lower()
if key in seen:
continue
seen.add(key)
try:
if resolved.exists() and resolved.is_dir():
dirs.append(resolved)
except Exception:
continue
return tuple(dirs)
def _iter_external_plugin_entries(plugin_dir: Path) -> Iterable[Tuple[str, Path, bool]]:
try:
children = sorted(plugin_dir.iterdir(), key=lambda entry: entry.name.lower())
except Exception:
return ()
out: List[Tuple[str, Path, bool]] = []
for child in children:
name = str(child.name or "").strip()
if not name or name.startswith("."):
continue
if child.is_file() and child.suffix.lower() == ".py" and child.stem != "__init__":
fingerprint = hashlib.sha1(str(child).encode("utf-8", errors="ignore")).hexdigest()[:10]
out.append((f"_medeia_plugin_{child.stem}_{fingerprint}", child, False))
continue
if child.is_dir():
init_py = child / "__init__.py"
if not init_py.exists() or not init_py.is_file():
continue
fingerprint = hashlib.sha1(str(child).encode("utf-8", errors="ignore")).hexdigest()[:10]
out.append((f"_medeia_plugin_pkg_{child.name}_{fingerprint}", init_py, True))
return tuple(out)
@dataclass(frozen=True)
class ProviderInfo:
"""Metadata about a single provider entry."""
"""Metadata about a single plugin entry."""
canonical_name: str
provider_class: Type[Provider]
@@ -56,14 +124,16 @@ class ProviderInfo:
class ProviderRegistry:
"""Handles discovery, registration, and lookup of provider classes."""
"""Handles discovery, registration, and lookup of built-in and external plugins."""
def __init__(self, package_name: str) -> None:
self.package_name = (package_name or "").strip()
self._infos: Dict[str, ProviderInfo] = {}
self._lookup: Dict[str, ProviderInfo] = {}
self._modules: set[str] = set()
self._external_modules: set[str] = set()
self._discovered = False
self._external_dirs_scanned = False
def _normalize(self, value: Any) -> str:
return str(value or "").strip().lower()
@@ -85,12 +155,10 @@ class ProviderRegistry:
if override_name:
_add(override_name)
else:
# Use explicit NAME or PROVIDER_NAME if available, else class name
_add(getattr(provider_class, "NAME", None))
_add(getattr(provider_class, "PROVIDER_NAME", None))
_add(getattr(provider_class, "PLUGIN_NAME", None))
_add(getattr(provider_class, "__name__", None))
for alias in getattr(provider_class, "PROVIDER_ALIASES", ()) or ():
for alias in getattr(provider_class, "PLUGIN_ALIASES", ()) or ():
_add(alias)
return names
@@ -104,14 +172,14 @@ class ProviderRegistry:
module_name: Optional[str] = None,
replace: bool = False,
) -> ProviderInfo:
"""Register a provider class with canonical and alias names."""
"""Register a plugin class with canonical and alias names."""
candidates = self._candidate_names(provider_class, override_name)
if not candidates:
raise ValueError("provider name candidates are required")
raise ValueError("plugin name candidates are required")
canonical = self._normalize(candidates[0])
if not canonical:
raise ValueError("provider name must not be empty")
raise ValueError("plugin name must not be empty")
alias_names: List[str] = []
alias_seen: set[str] = set()
@@ -165,7 +233,44 @@ class ProviderRegistry:
try:
self.register(candidate, module_name=module_name)
except Exception as exc:
log(f"[provider] Failed to register {module_name}.{candidate.__name__}: {exc}", file=sys.stderr)
log(f"[plugin] Failed to register {module_name}.{candidate.__name__}: {exc}", file=sys.stderr)
def _discover_external_plugins(self) -> None:
if self._external_dirs_scanned:
return
self._external_dirs_scanned = True
for plugin_dir in _iter_external_plugin_dirs():
try:
plugin_dir_str = str(plugin_dir)
if plugin_dir_str and plugin_dir_str not in sys.path:
sys.path.insert(0, plugin_dir_str)
except Exception:
pass
for module_name, module_path, is_package in _iter_external_plugin_entries(plugin_dir):
if module_name in self._external_modules:
continue
try:
if is_package:
spec = importlib.util.spec_from_file_location(
module_name,
str(module_path),
submodule_search_locations=[str(module_path.parent)],
)
else:
spec = importlib.util.spec_from_file_location(module_name, str(module_path))
if spec is None or spec.loader is None:
raise ImportError("missing module spec loader")
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
self._external_modules.add(module_name)
self._register_module(module)
except Exception as exc:
log(f"[plugin] Failed to load external plugin {module_path}: {exc}", file=sys.stderr)
def discover(self) -> None:
"""Import and register providers from the package."""
@@ -177,12 +282,13 @@ class ProviderRegistry:
try:
package = importlib.import_module(self.package_name)
except Exception as exc:
log(f"[provider] Failed to import package {self.package_name}: {exc}", file=sys.stderr)
log(f"[plugin] Failed to import package {self.package_name}: {exc}", file=sys.stderr)
return
self._register_module(package)
package_path = getattr(package, "__path__", None)
if not package_path:
self._discover_external_plugins()
return
for finder, module_name, _ in pkgutil.iter_modules(package_path):
@@ -194,18 +300,19 @@ class ProviderRegistry:
try:
module = importlib.import_module(module_path)
except Exception as exc:
log(f"[provider] Failed to load {module_path}: {exc}", file=sys.stderr)
log(f"[plugin] Failed to load {module_path}: {exc}", file=sys.stderr)
continue
self._register_module(module)
# Pick up any Provider subclasses loaded via other mechanisms.
self._sync_subclasses()
self._discover_external_plugins()
def _try_import_for_name(self, normalized_name: str) -> None:
"""Best-effort import for a single provider module.
"""Best-effort import for a single plugin module.
This avoids importing every provider module when the caller only needs
one provider (common for CLI usage).
one plugin (common for CLI usage).
"""
name = str(normalized_name or "").strip().lower()
if not name or not self.package_name:
@@ -249,6 +356,7 @@ class ProviderRegistry:
# module that matches the requested name.
if not self._discovered:
self._try_import_for_name(normalized)
self._discover_external_plugins()
info = self._lookup.get(normalized)
if info is not None:
return info
@@ -279,6 +387,9 @@ class ProviderRegistry:
_walk(Provider)
REGISTRY = ProviderRegistry("Provider")
PLUGIN_REGISTRY = REGISTRY
PluginInfo = ProviderInfo
PluginRegistry = ProviderRegistry
@lru_cache(maxsize=512)
@@ -289,18 +400,16 @@ def _provider_url_patterns(provider_class: Type[Provider]) -> Sequence[str]:
return []
def register_provider(
provider_class: Type[Provider],
def register_plugin(
plugin_class: Type[Provider],
*,
name: Optional[str] = None,
aliases: Optional[Sequence[str]] = None,
module_name: Optional[str] = None,
replace: bool = False,
) -> ProviderInfo:
"""Register a provider class from tests or third-party packages."""
return REGISTRY.register(
provider_class,
plugin_class,
override_name=name,
extra_aliases=aliases,
module_name=module_name,
@@ -308,7 +417,7 @@ def register_provider(
)
def get_provider_class(name: str) -> Optional[Type[Provider]]:
def get_plugin_class(name: str) -> Optional[Type[Provider]]:
info = REGISTRY.get(name)
if info is None:
return None
@@ -323,18 +432,18 @@ def selection_auto_stage_for_table(
if not t:
return None
provider_key = t.split(".", 1)[0] if "." in t else t
provider_class = get_provider_class(provider_key) or get_provider_class(t)
if provider_class is None:
plugin_key = t.split(".", 1)[0] if "." in t else t
plugin_class = get_plugin_class(plugin_key) or get_plugin_class(t)
if plugin_class is None:
return None
try:
return provider_class.selection_auto_stage(t, stage_args)
return plugin_class.selection_auto_stage(t, stage_args)
except Exception:
return None
def is_known_provider_name(name: str) -> bool:
def is_known_plugin_name(name: str) -> bool:
return REGISTRY.has_name(name)
@@ -406,83 +515,83 @@ def _collect_inline_choice_mapping(provider: Provider) -> Dict[str, List[Dict[st
return mapping
def get_provider(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[Provider]:
def get_plugin(name: str, config: Optional[Dict[str, Any]] = None) -> Optional[Provider]:
info = REGISTRY.get(name)
if info is None:
debug(f"[provider] Unknown provider: {name}")
debug(f"[plugin] Unknown plugin: {name}")
return None
try:
provider = info.provider_class(config)
if not provider.validate():
debug(f"[provider] Provider '{name}' is not available")
plugin = info.provider_class(config)
if not plugin.validate():
debug(f"[plugin] Plugin '{name}' is not available")
return None
return provider
return plugin
except Exception as exc:
debug(f"[provider] Error initializing '{name}': {exc}")
debug(f"[plugin] Error initializing '{name}': {exc}")
return None
def list_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
def list_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
try:
provider = info.provider_class(config)
availability[info.canonical_name] = provider.validate()
plugin = info.provider_class(config)
availability[info.canonical_name] = plugin.validate()
except Exception:
availability[info.canonical_name] = False
return availability
def get_search_provider(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]:
provider = get_provider(name, config)
if provider is None:
def get_search_plugin(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[SearchProvider]:
plugin = get_plugin(name, config)
if plugin is None:
return None
if not _supports_search(provider):
debug(f"[provider] Provider '{name}' does not support search")
if not _supports_search(plugin):
debug(f"[plugin] Plugin '{name}' does not support search")
return None
return provider # type: ignore[return-value]
return plugin # type: ignore[return-value]
def list_search_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
def list_search_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
try:
provider = info.provider_class(config)
plugin = info.provider_class(config)
availability[info.canonical_name] = bool(
provider.validate() and info.supports_search
plugin.validate() and info.supports_search
)
except Exception:
availability[info.canonical_name] = False
return availability
def get_file_provider(name: str,
def get_upload_plugin(name: str,
config: Optional[Dict[str, Any]] = None) -> Optional[FileProvider]:
provider = get_provider(name, config)
if provider is None:
plugin = get_plugin(name, config)
if plugin is None:
return None
if not _supports_upload(provider):
debug(f"[provider] Provider '{name}' does not support upload")
if not _supports_upload(plugin):
debug(f"[plugin] Plugin '{name}' does not support upload")
return None
return provider # type: ignore[return-value]
return plugin # type: ignore[return-value]
def list_file_providers(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
def list_upload_plugins(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
availability: Dict[str, bool] = {}
for info in REGISTRY.iter_providers():
try:
provider = info.provider_class(config)
plugin = info.provider_class(config)
availability[info.canonical_name] = bool(
provider.validate() and info.supports_upload
plugin.validate() and info.supports_upload
)
except Exception:
availability[info.canonical_name] = False
return availability
def match_provider_name_for_url(url: str) -> Optional[str]:
def match_plugin_name_for_url(url: str) -> Optional[str]:
raw_url = str(url or "").strip()
raw_url_lower = raw_url.lower()
try:
@@ -540,31 +649,31 @@ def match_provider_name_for_url(url: str) -> Optional[str]:
return None
def provider_inline_query_choices(
provider_name: str,
def plugin_inline_query_choices(
plugin_name: str,
field_name: str,
config: Optional[Dict[str, Any]] = None,
) -> List[str]:
"""Return provider-declared inline query choices for a field (e.g., system:GBA).
"""Return plugin-declared inline query choices for a field (e.g., system:GBA).
Providers can expose a mapping via ``QUERY_ARG_CHOICES`` (preferred) or
Plugins can expose a mapping via ``QUERY_ARG_CHOICES`` (preferred) or
``INLINE_QUERY_FIELD_CHOICES`` / ``inline_query_field_choices()``. The helper
keeps completion logic simple and reusable.
"""
pname = str(provider_name or "").strip().lower()
pname = str(plugin_name or "").strip().lower()
field = str(field_name or "").strip().lower()
if not pname or not field:
return []
provider = get_search_provider(pname, config)
if provider is None:
provider = get_provider(pname, config)
if provider is None:
plugin = get_search_plugin(pname, config)
if plugin is None:
plugin = get_plugin(pname, config)
if plugin is None:
return []
try:
mapping = _collect_inline_choice_mapping(provider)
mapping = _collect_inline_choice_mapping(plugin)
if not mapping:
return []
@@ -593,12 +702,32 @@ def provider_inline_query_choices(
return []
def get_provider_for_url(url: str,
config: Optional[Dict[str, Any]] = None) -> Optional[Provider]:
name = match_provider_name_for_url(url)
def get_plugin_for_url(url: str,
config: Optional[Dict[str, Any]] = None) -> Optional[Provider]:
name = match_plugin_name_for_url(url)
if not name:
return None
return get_provider(name, config)
return get_plugin(name, config)
def list_selection_url_prefixes() -> List[str]:
prefixes: List[str] = []
seen: set[str] = set()
for info in REGISTRY.iter_providers():
try:
values = info.provider_class.selection_url_prefixes()
except Exception:
values = ()
for value in values or ():
try:
normalized = str(value or "").strip().lower()
except Exception:
continue
if not normalized or normalized in seen:
continue
seen.add(normalized)
prefixes.append(normalized)
return prefixes
def resolve_inline_filters(
@@ -657,21 +786,25 @@ def resolve_inline_filters(
__all__ = [
"ProviderInfo",
"PluginInfo",
"Provider",
"SearchProvider",
"FileProvider",
"SearchResult",
"register_provider",
"get_provider",
"list_providers",
"get_search_provider",
"list_search_providers",
"get_file_provider",
"list_file_providers",
"match_provider_name_for_url",
"get_provider_for_url",
"get_provider_class",
"PluginRegistry",
"PLUGIN_REGISTRY",
"register_plugin",
"get_plugin",
"list_plugins",
"get_search_plugin",
"list_search_plugins",
"get_upload_plugin",
"list_upload_plugins",
"match_plugin_name_for_url",
"get_plugin_for_url",
"list_selection_url_prefixes",
"get_plugin_class",
"selection_auto_stage_for_table",
"download_soulseek_file",
"provider_inline_query_choices",
"plugin_inline_query_choices",
"is_known_plugin_name",
]
SYS/cmdlet_catalog.py
+15 -15
View File
@@ -5,6 +5,7 @@ from importlib import import_module, reload as reload_module
from types import ModuleType
from typing import Any, Dict, List, Optional
import logging
from ProviderCore.registry import get_plugin
logger = logging.getLogger(__name__)
try:
@@ -370,22 +371,21 @@ def get_cmdlet_arg_choices(
token = matrix_conf.get("access_token")
if hs and token:
try:
from Provider.matrix import Matrix
try:
m = Matrix(config)
rooms = m.list_rooms(room_ids=ids)
choices = []
for r in rooms or []:
name = str(r.get("name") or "").strip()
rid = str(r.get("room_id") or "").strip()
choices.append(name or rid)
if choices:
return choices
except Exception as exc:
logger.exception("Matrix provider failed while listing rooms: %s", exc)
provider = get_plugin("matrix", config)
if provider is not None:
try:
rooms = provider.list_rooms(room_ids=ids)
choices = []
for r in rooms or []:
name = str(r.get("name") or "").strip()
rid = str(r.get("room_id") or "").strip()
choices.append(name or rid)
if choices:
return choices
except Exception as exc:
logger.exception("Matrix provider failed while listing rooms: %s", exc)
except Exception as exc:
logger.exception("Failed to import Matrix provider or initialize: %s", exc)
logger.exception("Failed to initialize Matrix plugin: %s", exc)
except Exception as exc:
logger.exception("Failed to resolve matrix rooms: %s", exc)
SYS/cmdlet_spec.py
+16 -6
View File
@@ -90,10 +90,10 @@ class SharedArgs:
description="http parser",
)
PROVIDER = CmdletArg(
name="provider",
PLUGIN = CmdletArg(
name="plugin",
type="string",
description="selects provider",
description="selects plugin",
)
@staticmethod
@@ -284,7 +284,13 @@ class Cmdlet:
return {f"-{arg_name}", f"--{arg_name}"}
def build_flag_registry(self) -> Dict[str, set[str]]:
return {arg.name: self.get_flags(arg.name) for arg in self.arg}
registry: Dict[str, set[str]] = {}
for arg in self.arg:
try:
registry[arg.name] = {str(flag).lower() for flag in arg.to_flags()}
except Exception:
registry[arg.name] = {flag.lower() for flag in self.get_flags(arg.name)}
return registry
def parse_cmdlet_args(
@@ -335,8 +341,12 @@ def parse_cmdlet_args(
positional_args.append(spec)
arg_spec_map[canonical_key] = canonical_name
arg_spec_map[f"-{canonical_name}".lower()] = canonical_name
arg_spec_map[f"--{canonical_name}".lower()] = canonical_name
try:
for flag in spec.to_flags():
arg_spec_map[str(flag).lower()] = canonical_name
except Exception:
arg_spec_map[f"-{canonical_name}".lower()] = canonical_name
arg_spec_map[f"--{canonical_name}".lower()] = canonical_name
i = 0
positional_index = 0
SYS/metadata.py
+10 -29
View File
@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from ProviderCore.registry import get_plugin
from SYS.yt_metadata import extract_ytdlp_tags
try: # Optional; used when available for richer metadata fetches
@@ -2213,40 +2214,20 @@ def enrich_playlist_entries(entries: list, extractor: str) -> list:
Returns:
List of enriched entry dicts
"""
# Import here to avoid circular dependency
from tool.ytdlp import is_url_supported_by_ytdlp
if not entries:
return entries
enriched = []
for entry in entries:
# If entry has a direct URL, fetch its full metadata
entry_url = entry.get("url")
if entry_url and is_url_supported_by_ytdlp(entry_url):
try:
import yt_dlp
plugin = get_plugin("ytdlp", {})
if plugin is None:
return entries
ydl_opts: Any = {
"quiet": True,
"no_warnings": True,
"skip_download": True,
"noprogress": True,
"socket_timeout": 5,
"retries": 1,
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
full_info = ydl.extract_info(entry_url, download=False)
if full_info:
enriched.append(full_info)
continue
except Exception:
logger.exception("Failed to fetch full metadata for entry URL: %s", entry_url)
try:
enriched = plugin.enrich_playlist_entries(entries, extractor=extractor)
except Exception:
logger.exception("Failed to enrich playlist entries for extractor: %s", extractor)
return entries
# Fallback to original entry if fetch failed
enriched.append(entry)
return enriched
return enriched if isinstance(enriched, list) else entries
def format_playlist_entry(entry: Dict[str,
SYS/pipeline.py
+126 -79
View File
@@ -1505,9 +1505,9 @@ class PipelineExecutor:
"table") else None
)
# Prefer an explicit provider hint from table metadata when available.
# Prefer an explicit plugin hint from table metadata when available.
# This keeps @N selectors working even when row payloads don't carry a
# provider key (or when they carry a table-type like tidal.album).
# plugin key (or when they carry a table-type like tidal.album).
try:
meta = (
current_table.get_table_metadata()
@@ -1517,56 +1517,58 @@ class PipelineExecutor:
except Exception:
meta = None
if isinstance(meta, dict):
_add(meta.get("plugin"))
_add(meta.get("provider"))
except Exception:
logger.exception("Failed to inspect current_table/table metadata in _maybe_run_class_selector")
for item in selected_items or []:
if isinstance(item, dict):
_add(item.get("plugin"))
_add(item.get("provider"))
_add(item.get("store"))
_add(item.get("table"))
else:
_add(getattr(item, "plugin", None))
_add(getattr(item, "provider", None))
_add(getattr(item, "store", None))
_add(getattr(item, "table", None))
try:
from ProviderCore.registry import get_provider, is_known_provider_name
from ProviderCore.registry import get_plugin, is_known_plugin_name
except Exception:
get_provider = None # type: ignore
is_known_provider_name = None # type: ignore
get_plugin = None # type: ignore
is_known_plugin_name = None # type: ignore
# If we have a table-type like "tidal.album", also try its provider prefix ("tidal")
# when that prefix is a registered provider name.
if is_known_provider_name is not None:
# If we have a table-type like "tidal.album", also try its plugin prefix ("tidal")
# when that prefix is a registered plugin name.
if is_known_plugin_name is not None:
try:
for key in list(candidates):
if not isinstance(key, str):
continue
if "." not in key:
continue
if is_known_provider_name(key):
if is_known_plugin_name(key):
continue
prefix = str(key).split(".", 1)[0].strip().lower()
if prefix and is_known_provider_name(prefix):
if prefix and is_known_plugin_name(prefix):
_add(prefix)
except Exception:
logger.exception("Failed while computing provider prefix heuristics in _maybe_run_class_selector")
logger.exception("Failed while computing plugin prefix heuristics in _maybe_run_class_selector")
if get_provider is not None:
if get_plugin is not None:
for key in candidates:
try:
if is_known_provider_name is not None and (
not is_known_provider_name(key)):
if is_known_plugin_name is not None and (
not is_known_plugin_name(key)):
continue
except Exception:
# If the predicate fails for any reason, fall back to legacy behavior.
logger.exception("is_known_provider_name predicate failed for key %s; falling back", key)
logger.exception("is_known_plugin_name predicate failed for key %s; falling back", key)
try:
provider = get_provider(key, config)
provider = get_plugin(key, config)
except Exception as exc:
logger.exception("Failed to load provider '%s' during selector resolution: %s", key, exc)
logger.exception("Failed to load plugin '%s' during selector resolution: %s", key, exc)
continue
selector = getattr(provider, "selector", None)
if selector is None:
@@ -1583,6 +1585,92 @@ class PipelineExecutor:
if handled:
return True
@staticmethod
def _maybe_expand_plugin_selection(
selected_items: List[Any],
*,
ctx: Any,
config: Dict[str, Any],
stage_table: Any,
) -> Optional[List[Any]]:
candidates: list[str] = []
def _add(value: Any) -> None:
text = str(value or "").strip().lower()
if text and text not in candidates:
candidates.append(text)
table_type = None
try:
table_type = stage_table.table if stage_table is not None and hasattr(stage_table, "table") else None
except Exception:
table_type = None
_add(table_type)
try:
meta = (
stage_table.get_table_metadata()
if stage_table is not None and hasattr(stage_table, "get_table_metadata")
else getattr(stage_table, "table_metadata", None)
)
except Exception:
meta = None
if isinstance(meta, dict):
_add(meta.get("plugin"))
_add(meta.get("provider"))
for item in selected_items or []:
if isinstance(item, dict):
_add(item.get("plugin"))
_add(item.get("provider"))
_add(item.get("table"))
_add(item.get("source"))
else:
_add(getattr(item, "plugin", None))
_add(getattr(item, "provider", None))
_add(getattr(item, "table", None))
_add(getattr(item, "source", None))
try:
from ProviderCore.registry import get_plugin, is_known_plugin_name
except Exception:
return None
for key in list(candidates):
if "." in key:
prefix = str(key).split(".", 1)[0].strip().lower()
if prefix and prefix not in candidates:
candidates.append(prefix)
for key in candidates:
try:
if not is_known_plugin_name(key):
continue
except Exception:
continue
try:
plugin = get_plugin(key, config)
except Exception:
continue
if plugin is None:
continue
expand = getattr(plugin, "expand_selection", None)
if not callable(expand):
continue
try:
expanded = expand(
selected_items,
ctx=ctx,
stage_is_last=False,
table_type=str(table_type or ""),
)
except Exception:
logger.exception("%s expand_selection failed", key)
return None
if expanded:
return list(expanded)
return None
store_keys: list[str] = []
for item in selected_items or []:
if isinstance(item, dict):
@@ -1998,10 +2086,10 @@ class PipelineExecutor:
# IMPORTANT: Put selected row args *before* source_args.
# Rationale: The cmdlet argument parser treats the *first* unknown
# token as a positional value (e.g., URL). If `source_args`
# contain unknown flags (like -provider which download-file does
# contain unknown flags (like a removed legacy flag that download-file does
# not declare), they could be misinterpreted as the positional
# URL argument and cause attempts to download strings like
# "-provider" (which is invalid). By placing selection args
# not accept). By placing selection args
# first we ensure the intended URL/selection token is parsed
# as the positional URL and avoid this class of parsing errors.
expanded_stage: List[str] = cmd_list + selected_row_args + source_args
@@ -2081,66 +2169,15 @@ class PipelineExecutor:
print("No items matched selection in pipeline\n")
return False, None
# Provider selection expansion (non-terminal): allow certain provider tables
# (e.g. tidal.album) to expand to multiple downstream items when the user
# pipes into another stage (e.g. @N | .mpv or @N | add-file).
table_type_hint = None
try:
table_type_hint = (
stage_table.table
if stage_table is not None and hasattr(stage_table, "table")
else None
if stages:
expanded = PipelineExecutor._maybe_expand_plugin_selection(
filtered,
ctx=ctx,
config=config,
stage_table=stage_table,
)
except Exception:
table_type_hint = None
if stages and isinstance(table_type_hint, str) and table_type_hint.strip().lower() == "tidal.album":
try:
from ProviderCore.registry import get_provider
prov = get_provider("tidal", config)
except Exception:
prov = None
if prov is not None and hasattr(prov, "_extract_album_selection_context") and hasattr(prov, "_tracks_for_album"):
try:
album_contexts = prov._extract_album_selection_context(filtered) # type: ignore[attr-defined]
except Exception:
album_contexts = []
track_items: List[Any] = []
seen_track_ids: set[int] = set()
for album_id, album_title, artist_name in album_contexts or []:
try:
track_results = prov._tracks_for_album( # type: ignore[attr-defined]
album_id=album_id,
album_title=album_title,
artist_name=artist_name,
limit=500,
)
except Exception:
track_results = []
for tr in track_results or []:
try:
md = getattr(tr, "full_metadata", None)
tid = None
if isinstance(md, dict):
raw_id = md.get("trackId") or md.get("id")
try:
tid = int(raw_id) if raw_id is not None else None
except Exception:
tid = None
if tid is not None:
if tid in seen_track_ids:
continue
seen_track_ids.add(tid)
except Exception:
logger.exception("Failed to extract/parse track metadata in album processing")
track_items.append(tr)
if track_items:
filtered = track_items
table_type_hint = "tidal.track"
if expanded:
filtered = expanded
if PipelineExecutor._maybe_run_class_selector(
ctx,
@@ -2177,6 +2214,16 @@ class PipelineExecutor:
except Exception:
logger.exception("Failed to determine current_table for selection auto-insert; defaulting to None")
current_table = None
table_type_hint = None
try:
raw_table_type = (
stage_table.table
if stage_table is not None and hasattr(stage_table, "table") else None
)
if isinstance(raw_table_type, str) and raw_table_type.strip():
table_type_hint = raw_table_type
except Exception:
table_type_hint = None
table_type = None
try:
if isinstance(table_type_hint, str) and table_type_hint.strip():
SYS/result_table_adapters.py
+23 -29
View File
@@ -1,10 +1,4 @@
"""Provider registry for ResultTable API (breaking, strict API).
Providers register themselves here with an adapter and optional column factory
and selection function. Consumers (cmdlets) can look up providers by name and
obtain the columns and selection behavior for building tables and for selection
args used by subsequent cmdlets.
"""
"""Plugin registry for the strict ResultTable API."""
from __future__ import annotations
from dataclasses import dataclass
@@ -18,7 +12,7 @@ SelectionFn = Callable[[ResultModel], List[str]]
@dataclass
class Provider:
class Plugin:
name: str
adapter: ProviderAdapter
# columns can be a static list or a factory that derives columns from sample rows
@@ -28,7 +22,7 @@ class Provider:
def get_columns(self, rows: Optional[Iterable[ResultModel]] = None) -> List[ColumnSpec]:
if self.columns is None:
raise ValueError(f"provider '{self.name}' must define columns")
raise ValueError(f"plugin '{self.name}' must define columns")
if callable(self.columns):
rows_list = list(rows) if rows is not None else []
@@ -37,13 +31,13 @@ class Provider:
cols = list(self.columns)
if not cols:
raise ValueError(f"provider '{self.name}' produced no columns")
raise ValueError(f"plugin '{self.name}' produced no columns")
return cols
def selection_args(self, row: ResultModel) -> List[str]:
if not callable(self.selection_fn):
raise ValueError(f"provider '{self.name}' must define a selection function")
raise ValueError(f"plugin '{self.name}' must define a selection function")
sel = list(self.selection_fn(ensure_result_model(row)))
return sel
@@ -54,7 +48,7 @@ class Provider:
try:
rows = [ensure_result_model(r) for r in self.adapter(items)]
except Exception as exc:
raise RuntimeError(f"provider '{self.name}' adapter failed") from exc
raise RuntimeError(f"plugin '{self.name}' adapter failed") from exc
cols = self.get_columns(rows)
return ResultTable(provider=self.name, rows=rows, columns=cols, meta=self.metadata or {})
@@ -82,37 +76,37 @@ class Provider:
return [self.serialize_row(r) for r in rows]
_PROVIDERS: Dict[str, Provider] = {}
_PLUGINS: Dict[str, Plugin] = {}
def register_provider(
def register_plugin(
name: str,
adapter: ProviderAdapter,
*,
columns: Union[List[ColumnSpec], ColumnFactory],
selection_fn: SelectionFn,
metadata: Optional[Dict[str, Any]] = None,
) -> Provider:
) -> Plugin:
name = str(name or "").strip().lower()
if not name:
raise ValueError("provider name required")
if name in _PROVIDERS:
raise ValueError(f"provider already registered: {name}")
raise ValueError("plugin name required")
if name in _PLUGINS:
raise ValueError(f"plugin already registered: {name}")
if columns is None:
raise ValueError("provider registration requires columns")
raise ValueError("plugin registration requires columns")
if selection_fn is None:
raise ValueError("provider registration requires selection_fn")
p = Provider(name=name, adapter=adapter, columns=columns, selection_fn=selection_fn, metadata=metadata)
_PROVIDERS[name] = p
return p
raise ValueError("plugin registration requires selection_fn")
plugin = Plugin(name=name, adapter=adapter, columns=columns, selection_fn=selection_fn, metadata=metadata)
_PLUGINS[name] = plugin
return plugin
def get_provider(name: str) -> Provider:
def get_plugin(name: str) -> Plugin:
normalized = str(name or "").lower()
if normalized not in _PROVIDERS:
raise KeyError(f"provider not registered: {name}")
return _PROVIDERS[normalized]
if normalized not in _PLUGINS:
raise KeyError(f"plugin not registered: {name}")
return _PLUGINS[normalized]
def list_providers() -> List[str]:
return list(_PROVIDERS.keys())
def list_plugins() -> List[str]:
return list(_PLUGINS.keys())
SYS/rich_display.py
+3 -3
View File
@@ -148,7 +148,7 @@ def show_store_config_panel(
def show_available_providers_panel(provider_names: List[str]) -> None:
"""Show a Rich panel listing available/configured providers."""
"""Show a Rich panel listing available/configured plugins."""
from rich.columns import Columns
from rich.console import Group
@@ -164,13 +164,13 @@ def show_available_providers_panel(provider_names: List[str]) -> None:
)
group = Group(
Text("The following providers are configured and ready to use:\n"),
Text("The following plugins are configured and ready to use:\n"),
cols
)
panel = Panel(
group,
title="[bold green]Configured Providers[/bold green]",
title="[bold green]Configured Plugins[/bold green]",
border_style="green",
padding=(1, 2)
)
Store/HydrusNetwork.py
+9 -5
View File
@@ -526,9 +526,6 @@ class HydrusNetwork(Store):
# Upload file if not already present
if not file_exists:
debug(
f"{self._log_prefix()} Uploading: {file_path.name}"
)
response = client.add_file(file_path)
# Extract hash from response
@@ -553,8 +550,15 @@ class HydrusNetwork(Store):
hydrus_hash = None
if not hydrus_hash or len(str(hydrus_hash)) != 64:
debug(
f"{self._log_prefix()} Hydrus response hash missing/invalid; using precomputed hash"
debug_panel(
"Hydrus upload fallback",
[
("store", self.NAME),
("file", file_path.name),
("reason", "response hash missing/invalid"),
("fallback_hash", file_hash),
],
border_style="yellow",
)
hydrus_hash = file_hash
TUI.py
+2 -2
View File
@@ -613,14 +613,14 @@ class PipelineHubApp(App):
# Run startup check automatically
self._run_pipeline_background(".status")
# Provide a visible startup summary of configured providers/stores for debugging
# Provide a visible startup summary of configured plugins/stores for debugging
try:
cfg = load_config() or {}
provs = list(cfg.get("provider", {}).keys()) if isinstance(cfg.get("provider"), dict) else []
stores = list(cfg.get("store", {}).keys()) if isinstance(cfg.get("store"), dict) else []
prov_display = ", ".join(provs[:10]) + ("..." if len(provs) > 10 else "")
store_display = ", ".join(stores[:10]) + ("..." if len(stores) > 10 else "")
self._append_log_line(f"Startup config: providers={len(provs)} ({prov_display or '(none)'}), stores={len(stores)} ({store_display or '(none)'}), db={db.db_path.name}")
self._append_log_line(f"Startup config: plugins={len(provs)} ({prov_display or '(none)'}), stores={len(stores)} ({store_display or '(none)'}), db={db.db_path.name}")
except Exception:
logger.exception("Failed to produce startup config summary")
TUI/modalscreen/config_modal.py
+26 -25
View File
@@ -23,7 +23,7 @@ from SYS.config import (
from SYS.database import db
from SYS.logger import log, debug
from Store.registry import _discover_store_classes, _required_keys_for
from ProviderCore.registry import list_providers
from ProviderCore.registry import get_plugin, list_plugins
from TUI.modalscreen.matrix_room_picker import MatrixRoomPicker
from TUI.modalscreen.selection_modal import SelectionModal
import logging
@@ -177,7 +177,7 @@ class ConfigModal(ModalScreen):
with ListView(id="category-list"):
yield ListItem(Label("Global Settings"), id="cat-globals")
yield ListItem(Label("Stores"), id="cat-stores")
yield ListItem(Label("Providers"), id="cat-providers")
yield ListItem(Label("Plugins"), id="cat-providers")
yield ListItem(Label("Tools"), id="cat-tools")
with Vertical(id="config-content"):
@@ -187,7 +187,7 @@ class ConfigModal(ModalScreen):
# Durable synchronous save: waits and verifies DB persisted critical keys
yield Button("Save (durable)", variant="primary", id="save-durable-btn")
yield Button("Add Store", variant="primary", id="add-store-btn")
yield Button("Add Provider", variant="primary", id="add-provider-btn")
yield Button("Add Plugin", variant="primary", id="add-provider-btn")
yield Button("Add Tool", variant="primary", id="add-tool-btn")
yield Button("Back", id="back-btn")
yield Button("Close", variant="error", id="cancel-btn")
@@ -381,10 +381,10 @@ class ConfigModal(ModalScreen):
container.mount(row)
def render_providers(self, container: ScrollableContainer) -> None:
container.mount(Label("Configured Providers", classes="config-label"))
container.mount(Label("Configured Plugins", classes="config-label"))
providers = self.config_data.get("provider", {})
if not providers:
container.mount(Static("No providers configured."))
container.mount(Static("No plugins configured."))
else:
for i, (name, _) in enumerate(providers.items()):
edit_id = f"edit-provider-{i}"
@@ -448,9 +448,9 @@ class ConfigModal(ModalScreen):
# Fetch Provider schema
if item_type == "provider":
from ProviderCore.registry import get_provider_class
from ProviderCore.registry import get_plugin_class
try:
pcls = get_provider_class(item_name)
pcls = get_plugin_class(item_name)
if pcls and hasattr(pcls, "config_schema") and callable(pcls.config_schema):
for field_def in pcls.config_schema():
k = field_def.get("key")
@@ -625,9 +625,9 @@ class ConfigModal(ModalScreen):
# If it's a provider, we might have required keys (legacy check fallback)
if item_type == "provider":
# 2. Legacy required_config_keys
from ProviderCore.registry import get_provider_class
from ProviderCore.registry import get_plugin_class
try:
pcls = get_provider_class(item_name)
pcls = get_plugin_class(item_name)
if pcls:
required_keys = pcls.required_config_keys()
for rk in required_keys:
@@ -886,18 +886,18 @@ class ConfigModal(ModalScreen):
logger.exception("Failed to inspect store class config_schema for '%s'", stype)
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
elif bid == "add-provider-btn":
provider_names = list(list_providers().keys())
provider_names = list(list_plugins().keys())
options = []
from ProviderCore.registry import get_provider_class
from ProviderCore.registry import get_plugin_class
for ptype in provider_names:
pcls = get_provider_class(ptype)
pcls = get_plugin_class(ptype)
if pcls and hasattr(pcls, "config_schema") and callable(pcls.config_schema):
try:
if pcls.config_schema():
options.append(ptype)
except Exception:
logger.exception("Failed to inspect provider class config_schema for '%s'", ptype)
self.app.push_screen(SelectionModal("Select Provider Type", options), callback=self.on_provider_type_selected)
self.app.push_screen(SelectionModal("Select Plugin Type", options), callback=self.on_provider_type_selected)
elif bid == "add-tool-btn":
# Discover tool modules that advertise a config_schema()
options = []
@@ -1067,9 +1067,9 @@ class ConfigModal(ModalScreen):
# For providers, they are usually top-level entries in 'provider' dict
if ptype not in self.config_data["provider"]:
from ProviderCore.registry import get_provider_class
from ProviderCore.registry import get_plugin_class
try:
pcls = get_provider_class(ptype)
pcls = get_plugin_class(ptype)
new_config = {}
if pcls:
# Use schema for defaults
@@ -1273,9 +1273,9 @@ class ConfigModal(ModalScreen):
@work(thread=True)
def _matrix_test_background(self) -> None:
try:
from Provider.matrix import Matrix
provider = Matrix(self.config_data)
provider = get_plugin("matrix", self.config_data)
if provider is None:
raise RuntimeError("Matrix plugin unavailable")
rooms = provider.list_rooms()
self.app.call_from_thread(self._matrix_test_result, True, rooms, None)
except Exception as exc:
@@ -1433,9 +1433,9 @@ class ConfigModal(ModalScreen):
@work(thread=True)
def _matrix_load_background(self) -> None:
try:
from Provider.matrix import Matrix
provider = Matrix(self.config_data)
provider = get_plugin("matrix", self.config_data)
if provider is None:
raise RuntimeError("Matrix plugin unavailable")
rooms = provider.list_rooms()
self.app.call_from_thread(self._matrix_load_result, True, rooms, None)
except Exception as exc:
@@ -1626,8 +1626,9 @@ class ConfigModal(ModalScreen):
return []
try:
from Provider.matrix import Matrix
provider = Matrix(self.config_data)
provider = get_plugin("matrix", self.config_data)
if provider is None:
return []
rooms = provider.list_rooms(room_ids=ids_list)
return rooms or []
except Exception as exc:
@@ -1870,9 +1871,9 @@ class ConfigModal(ModalScreen):
required_keys = list(_required_keys_for(classes[stype]))
section = self.config_data.get("store", {}).get(stype, {}).get(item_name, {})
elif item_type == "provider":
from ProviderCore.registry import get_provider_class
from ProviderCore.registry import get_plugin_class
try:
pcls = get_provider_class(item_name)
pcls = get_plugin_class(item_name)
if pcls:
# Collect required keys from schema
if hasattr(pcls, "config_schema") and callable(pcls.config_schema):
TUI/modalscreen/matrix_room_picker.py
+4 -2
View File
@@ -8,6 +8,7 @@ from textual.screen import ModalScreen
from textual.widgets import Static, Button, Checkbox, ListView, ListItem
from textual import work
from rich.text import Text
from ProviderCore.registry import get_plugin
import logging
logger = logging.getLogger(__name__)
@@ -181,8 +182,9 @@ class MatrixRoomPicker(ModalScreen[List[str]]):
@work(thread=True)
def _load_rooms_background(self) -> None:
try:
from Provider.matrix import Matrix
provider = Matrix(self.config)
provider = get_plugin("matrix", self.config)
if provider is None:
raise RuntimeError("Matrix plugin unavailable")
rooms = provider.list_rooms()
self.app.call_from_thread(self._apply_room_results, rooms, None)
except Exception as exc:
TUI/modalscreen/search.py
+3 -3
View File
@@ -16,7 +16,7 @@ import asyncio
sys.path.insert(0, str(Path(__file__).parent.parent))
from SYS.config import load_config, resolve_output_dir
from SYS.result_table import Table
from ProviderCore.registry import get_search_provider
from ProviderCore.registry import get_search_plugin
logger = logging.getLogger(__name__)
@@ -174,7 +174,7 @@ class SearchModal(ModalScreen):
self.current_worker.log_step(f"Connecting to {source}...")
try:
provider = get_search_provider(source)
provider = get_search_plugin(source)
if not provider:
logger.error(f"[search-modal] Provider not available: {source}")
if self.current_worker:
@@ -380,7 +380,7 @@ class SearchModal(ModalScreen):
config = load_config()
output_dir = resolve_output_dir(config)
provider = get_search_provider("openlibrary", config=config)
provider = get_search_plugin("openlibrary", config=config)
if not provider:
logger.error("[search-modal] Provider not available: openlibrary")
return
cmdlet/_shared.py
+18 -21
View File
@@ -199,10 +199,10 @@ class SharedArgs:
type="string",
description="http parser",
)
PROVIDER = CmdletArg(
name="provider",
PLUGIN = CmdletArg(
name="plugin",
type="string",
description="selects provider",
description="selects plugin",
)
@staticmethod
@@ -538,10 +538,13 @@ class Cmdlet:
elif low in flags.get('tag', set()):
# handle tag
"""
return {
arg.name: self.get_flags(arg.name)
for arg in self.arg
}
registry: Dict[str, set[str]] = {}
for arg in self.arg:
try:
registry[arg.name] = {str(flag).lower() for flag in arg.to_flags()}
except Exception:
registry[arg.name] = {flag.lower() for flag in self.get_flags(arg.name)}
return registry
# Tag groups cache (loaded from JSON config file)
@@ -642,10 +645,14 @@ def parse_cmdlet_args(args: Sequence[str],
else:
flagged_args.append(spec)
# Register all prefix variants for flagged lookup
arg_spec_map[canonical_name.lower()] = canonical_name # bare name
arg_spec_map[f"-{canonical_name}".lower()] = canonical_name # single dash
arg_spec_map[f"--{canonical_name}".lower()] = canonical_name # double dash
# Register all supported flag variants, including legacy aliases.
arg_spec_map[canonical_name.lower()] = canonical_name # bare canonical name
try:
for flag in spec.to_flags():
arg_spec_map[str(flag).lower()] = canonical_name
except Exception:
arg_spec_map[f"-{canonical_name}".lower()] = canonical_name
arg_spec_map[f"--{canonical_name}".lower()] = canonical_name
# Parse arguments
i = 0
@@ -3143,16 +3150,6 @@ def register_url_with_local_library(
"""
# Folder store removed; local library URL registration is disabled.
return False
try:
# Provider-specific implementation lives with the provider code.
from Provider.tidal_manifest import resolve_tidal_manifest_path
except Exception: # pragma: no cover
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
_ = item
return None
def check_url_exists_in_storage(
urls: Sequence[str],
storage: Any,
cmdlet/add_file.py
+79 -62
View File
@@ -176,14 +176,14 @@ class Add_File(Cmdlet):
super().__init__(
name="add-file",
summary=
"Ingest a local media file to a store backend, file provider, or local directory.",
"Ingest a local media file to a store backend, upload plugin, or local directory.",
usage=
"add-file (-path <filepath> | <piped>) (-storage <location> | -provider <fileprovider>) [-delete]",
"add-file (-path <filepath> | <piped>) (-storage <location> | -plugin <upload-plugin>) [-delete]",
arg=[
SharedArgs.PATH,
SharedArgs.STORE,
SharedArgs.URL,
SharedArgs.PROVIDER,
SharedArgs.PLUGIN,
CmdletArg(
name="delete",
type="flag",
@@ -198,7 +198,7 @@ class Add_File(Cmdlet):
" hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory",
" <path>: Copy file to specified directory",
"- File provider options (use -provider):",
"- Upload plugin options (use -plugin):",
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
@@ -224,13 +224,13 @@ class Add_File(Cmdlet):
path_arg = parsed.get("path")
location = parsed.get("store")
source_url_arg = parsed.get("url")
provider_name = parsed.get("provider")
plugin_name = parsed.get("plugin")
delete_after = parsed.get("delete", False)
# Convenience: when piping a file into add-file, allow `-path <existing dir>`
# to act as the destination export directory.
# Example: screen-shot "https://..." | add-file -path "C:\Users\Admin\Desktop"
if path_arg and not location and not provider_name:
if path_arg and not location and not plugin_name:
try:
candidate_dir = Path(str(path_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
@@ -263,7 +263,7 @@ class Add_File(Cmdlet):
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
if path_arg and location and not provider_name:
if path_arg and location and not plugin_name:
# Support comma-separated path lists: -path "file1,file2,file3"
# This is the mechanism used by @N expansion for directory tables.
try:
@@ -403,7 +403,7 @@ class Add_File(Cmdlet):
("result_type", type(result).__name__),
("items", total_items),
("location", location),
("provider", provider_name),
("plugin", plugin_name),
("delete", delete_after),
],
border_style="cyan",
@@ -599,8 +599,8 @@ class Add_File(Cmdlet):
export_destination=(Path(location) if location and not is_storage_backend_location else None),
store_instance=storage_registry,
)
if not media_path and provider_name:
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source(
if not media_path and plugin_name:
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source(
pipe_obj, config, storage_registry
)
if media_path:
@@ -610,7 +610,7 @@ class Add_File(Cmdlet):
[
("path", media_path),
("hash", file_hash or "N/A"),
("provider", provider_name or "local"),
("plugin", plugin_name or "local"),
],
border_style="green",
)
@@ -635,10 +635,10 @@ class Add_File(Cmdlet):
progress.step("hashing file")
progress.step("ingesting file")
if provider_name:
code = self._handle_provider_upload(
if plugin_name:
code = self._handle_plugin_upload(
media_path,
provider_name,
plugin_name,
pipe_obj,
config,
delete_after_item
@@ -1365,7 +1365,7 @@ class Add_File(Cmdlet):
hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None)
return candidate, hash_hint, None
downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_provider_result(
downloaded_path, hash_hint, tmp_dir = Add_File._maybe_download_plugin_result(
result,
pipe_obj,
config,
@@ -1393,45 +1393,41 @@ class Add_File(Cmdlet):
return normalized
@staticmethod
def _maybe_download_provider_result(
def _maybe_download_plugin_result(
result: Any,
pipe_obj: models.PipeObject,
config: Dict[str, Any],
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
provider_key = None
plugin_key = None
for source in (
pipe_obj.provider,
get_field(result, "plugin"),
get_field(result, "provider"),
get_field(result, "table"),
):
candidate = Add_File._normalize_provider_key(source)
if candidate:
provider_key = candidate
plugin_key = candidate
break
if not provider_key:
if not plugin_key:
return None, None, None
provider = get_search_provider(provider_key, config)
if provider is None:
from ProviderCore.registry import get_search_plugin
plugin = get_search_plugin(plugin_key, config)
if plugin is None:
return None, None, None
# Check for specialized download helper (used by AllDebrid and potentially others)
handler = getattr(provider, "download_for_pipe_result", None)
if not callable(handler):
# Fallback: check class if it's a classmethod and instance didn't have it (unlikely but safe)
handler = getattr(type(provider), "download_for_pipe_result", None)
if callable(handler):
try:
return handler(result, pipe_obj, config)
except Exception as exc:
debug(f"[add-file] Provider '{provider_key}' download helper failed: {exc}")
try:
return plugin.resolve_pipe_result_download(result, pipe_obj)
except Exception as exc:
debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}")
return None, None, None
@staticmethod
def _download_provider_source(
def _download_piped_source(
pipe_obj: models.PipeObject,
config: Dict[str, Any],
store_instance: Optional[Any],
@@ -2152,23 +2148,23 @@ class Add_File(Cmdlet):
return 0
@staticmethod
def _handle_provider_upload(
def _handle_plugin_upload(
media_path: Path,
provider_name: str,
plugin_name: str,
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
delete_after: bool,
) -> int:
"""Handle uploading to a file provider (e.g. 0x0)."""
from ProviderCore.registry import get_file_provider
"""Handle uploading via an upload plugin (e.g. 0x0)."""
from ProviderCore.registry import get_upload_plugin
log(f"Uploading via {provider_name}: {media_path.name}", file=sys.stderr)
log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr)
try:
file_provider = get_file_provider(provider_name, config)
file_provider = get_upload_plugin(plugin_name, config)
if not file_provider:
log(f"File provider '{provider_name}' not available", file=sys.stderr)
log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr)
return 1
hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj)
@@ -2183,8 +2179,8 @@ class Add_File(Cmdlet):
# Update PipeObject and emit
extra_updates: Dict[str,
Any] = {
"provider": provider_name,
"provider_url": hoster_url,
"plugin": plugin_name,
"plugin_url": hoster_url,
}
if isinstance(pipe_obj.extra, dict):
# Also track hoster URL as a url for downstream steps
@@ -2197,7 +2193,7 @@ class Add_File(Cmdlet):
Add_File._update_pipe_object_destination(
pipe_obj,
hash_value=f_hash or "unknown",
store=provider_name or "provider",
store=plugin_name or "plugin",
path=file_path,
tag=pipe_obj.tag,
title=pipe_obj.title or (media_path.name if media_path else None),
@@ -2445,9 +2441,6 @@ class Add_File(Cmdlet):
try:
adder = getattr(backend, "add_tag", None)
if callable(adder):
debug(
f"[add-file] Applying {len(tags)} tag(s) post-upload to {backend_name}"
)
adder(resolved_hash, list(tags))
except Exception as exc:
log(f"[add-file] Post-upload tagging failed for {backend_name}: {exc}", file=sys.stderr)
@@ -2479,48 +2472,72 @@ class Add_File(Cmdlet):
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
debug(
f"[add-file] Writing sub note (len={len(str(sub_note))}) to {backend_name}:{resolved_hash}"
)
setter(resolved_hash, "sub", sub_note)
except Exception as exc:
debug(f"[add-file] sub note write failed: {exc}")
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "sub"),
("error", exc),
],
border_style="yellow",
)
lyric_note = Add_File._get_note_text(result, pipe_obj, "lyric")
if lyric_note:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
debug(
f"[add-file] Writing lyric note (len={len(str(lyric_note))}) to {backend_name}:{resolved_hash}"
)
setter(resolved_hash, "lyric", lyric_note)
except Exception as exc:
debug(f"[add-file] lyric note write failed: {exc}")
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "lyric"),
("error", exc),
],
border_style="yellow",
)
chapters_note = Add_File._get_note_text(result, pipe_obj, "chapters")
if chapters_note:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
debug(
f"[add-file] Writing chapters note (len={len(str(chapters_note))}) to {backend_name}:{resolved_hash}"
)
setter(resolved_hash, "chapters", chapters_note)
except Exception as exc:
debug(f"[add-file] chapters note write failed: {exc}")
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "chapters"),
("error", exc),
],
border_style="yellow",
)
caption_note = Add_File._get_note_text(result, pipe_obj, "caption")
if caption_note:
try:
setter = getattr(backend, "set_note", None)
if callable(setter):
debug(
f"[add-file] Writing caption note (len={len(str(caption_note))}) to {backend_name}:{resolved_hash}"
)
setter(resolved_hash, "caption", caption_note)
except Exception as exc:
debug(f"[add-file] caption note write failed: {exc}")
debug_panel(
"add-file note write failed",
[
("store", backend_name),
("hash", resolved_hash),
("note", "caption"),
("error", exc),
],
border_style="yellow",
)
meta: Dict[str,
Any] = {}
cmdlet/download_file.py
+205 -2023
View File
File diff suppressed because it is too large Load Diff
cmdlet/get_tag.py
+141 -869
View File
File diff suppressed because it is too large Load Diff
cmdlet/merge_file.py
+13 -47
View File
@@ -154,47 +154,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if urls_to_download and len(urls_to_download) >= 2:
try:
# Compute a batch hint (audio vs video + single-format id) once.
mode_hint: Optional[str] = None
forced_format: Optional[str] = None
try:
from tool.ytdlp import YtDlpTool, list_formats
sample_url = urls_to_download[0]
cookiefile = None
try:
cookie_path = YtDlpTool(config).resolve_cookiefile()
if cookie_path is not None and cookie_path.is_file():
cookiefile = str(cookie_path)
except Exception:
cookiefile = None
fmts = list_formats(
sample_url,
no_playlist=False,
playlist_items=None,
cookiefile=cookiefile
)
if isinstance(fmts, list) and fmts:
has_video = False
for f in fmts:
if not isinstance(f, dict):
continue
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
if vcodec and vcodec != "none":
has_video = True
break
mode_hint = "video" if has_video else "audio"
if len(fmts) == 1 and isinstance(fmts[0], dict):
fid = str(fmts[0].get("format_id") or "").strip()
if fid:
forced_format = fid
except Exception:
mode_hint = None
forced_format = None
from cmdlet.download_file import Download_File
from ProviderCore.registry import get_plugin_for_url
expanded: List[Dict[str, Any]] = []
downloaded_any = False
@@ -207,12 +167,18 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
expanded.append(it)
continue
downloaded = Download_File.download_streaming_url_as_pipe_objects(
u,
config,
mode_hint=mode_hint,
ytdl_format_hint=forced_format,
)
downloaded = []
try:
plugin = get_plugin_for_url(u, config)
except Exception:
plugin = None
if plugin is not None and hasattr(plugin, "download_url_as_pipe_objects"):
try:
downloaded = plugin.download_url_as_pipe_objects(u)
except TypeError:
downloaded = plugin.download_url_as_pipe_objects(u, output_dir=None)
except Exception:
downloaded = []
if downloaded:
expanded.extend(downloaded)
downloaded_any = True
cmdlet/provider_table.py
+15 -15
View File
@@ -7,7 +7,7 @@ from . import _shared as sh
from SYS.logger import log
from SYS import pipeline as ctx
from SYS.result_table_adapters import get_provider
from SYS.result_table_adapters import get_plugin
from SYS.result_table_renderers import RichRenderer
Cmdlet = sh.Cmdlet
@@ -16,19 +16,19 @@ parse_cmdlet_args = sh.parse_cmdlet_args
CMDLET = Cmdlet(
name="provider-table",
summary="Render a provider's result set and optionally run a follow-up cmdlet using the selected row.",
usage="provider-table -provider <name> [-sample] [-select <n>] [-run-cmd <name>]",
name="plugin-table",
summary="Render a plugin's result set and optionally run a follow-up cmdlet using the selected row.",
usage="plugin-table -plugin <name> [-sample] [-select <n>] [-run-cmd <name>]",
arg=[
CmdletArg("provider", type="string", description="Provider name to render (default: example)"),
CmdletArg("sample", type="flag", description="Use provider sample/demo items when available."),
CmdletArg("plugin", type="string", description="Plugin name to render (default: example)"),
CmdletArg("sample", type="flag", description="Use plugin sample/demo items when available."),
CmdletArg("select", type="int", description="1-based row index to select and use for follow-up command."),
CmdletArg("run-cmd", type="string", description="Cmdlet to invoke with the selected row's selector args."),
],
detail=[
"Use a registered provider to build a table and optionally run another cmdlet with selection args.",
"Use a registered plugin to build a table and optionally run another cmdlet with selection args.",
"Emits pipeline-friendly dicts enriched with `_selection_args` so you can use @N syntax to select and chain.",
"Example: provider-table -provider example -sample | @1 | add-file -store my_store",
"Example: plugin-table -plugin example -sample | @1 | add-file -store my_store",
],
)
@@ -36,15 +36,15 @@ CMDLET = Cmdlet(
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
parsed = parse_cmdlet_args(args, CMDLET)
provider_name = parsed.get("provider") or "example"
plugin_name = parsed.get("plugin") or "example"
use_sample = bool(parsed.get("sample", False))
run_cmd = parsed.get("run-cmd")
select_raw = parsed.get("select")
try:
provider = get_provider(provider_name)
provider = get_plugin(plugin_name)
except Exception:
log(f"Unknown provider: {provider_name}", file=sys.stderr)
log(f"Unknown plugin: {plugin_name}", file=sys.stderr)
return 1
# Obtain items to feed to the adapter
@@ -55,23 +55,23 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
mod = __import__(provider.adapter.__module__, fromlist=["*"])
items = getattr(mod, "SAMPLE_ITEMS", None)
if items is None:
log("Provider does not expose SAMPLE_ITEMS; no sample available", file=sys.stderr)
log("Plugin does not expose SAMPLE_ITEMS; no sample available", file=sys.stderr)
return 1
except Exception:
log("Failed to load provider sample", file=sys.stderr)
log("Failed to load plugin sample", file=sys.stderr)
return 1
else:
# Require input for non-sample runs
inputs = list(result) if isinstance(result, Iterable) else []
if not inputs:
log("No input provided. Use -sample for demo or pipe provider items in.", file=sys.stderr)
log("No input provided. Use -sample for demo or pipe plugin items in.", file=sys.stderr)
return 1
items = inputs
try:
table = provider.build_table(items)
except Exception as exc:
log(f"Provider '{provider.name}' failed: {exc}", file=sys.stderr)
log(f"Plugin '{provider.name}' failed: {exc}", file=sys.stderr)
return 1
# Emit rows for downstream pipeline consumption (pipable behavior).
cmdlet/search_file.py
+31 -85
View File
@@ -15,7 +15,7 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import log, debug
from SYS.payload_builders import build_file_result_payload, normalize_file_extension
from ProviderCore.registry import get_search_provider, list_search_providers
from ProviderCore.registry import get_search_plugin, list_search_plugins
from SYS.rich_display import (
show_provider_config_panel,
show_store_config_panel,
@@ -169,8 +169,8 @@ class search_file(Cmdlet):
def __init__(self) -> None:
super().__init__(
name="search-file",
summary="Search storage backends (Hydrus) or external providers (via -provider).",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-provider NAME]",
summary="Search storage backends (Hydrus) or external plugins (via -plugin).",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-plugin NAME]",
arg=[
CmdletArg(
"limit",
@@ -179,11 +179,7 @@ class search_file(Cmdlet):
),
SharedArgs.STORE,
SharedArgs.QUERY,
CmdletArg(
"provider",
type="string",
description="External provider name (e.g., tidal, youtube, soulseek, etc)",
),
SharedArgs.PLUGIN,
CmdletArg(
"open",
type="integer",
@@ -209,10 +205,10 @@ class search_file(Cmdlet):
"search-file 'example.com/path' -query 'ext:pdf' # Web: site:example.com filetype:pdf",
"search-file -query 'site:example.com filetype:epub history' # Web: site-scoped search",
"",
"Provider search (-provider):",
"search-file -provider youtube 'tutorial' # Search YouTube provider",
"search-file -provider alldebrid '*' # List AllDebrid magnets",
"search-file -provider alldebrid -open 123 '*' # Show files for a magnet",
"Plugin search (-plugin):",
"search-file -plugin youtube 'tutorial' # Search YouTube plugin",
"search-file -plugin alldebrid '*' # List AllDebrid magnets",
"search-file -plugin alldebrid -open 123 '*' # Show files for a magnet",
],
exec=self.run,
)
@@ -1451,10 +1447,10 @@ class search_file(Cmdlet):
self._set_storage_display_columns(payload)
return payload
def _run_provider_search(
def _run_plugin_search(
self,
*,
provider_name: str,
plugin_name: str,
query: str,
limit: int,
limit_set: bool,
@@ -1463,9 +1459,9 @@ class search_file(Cmdlet):
refresh_mode: bool,
config: Dict[str, Any],
) -> int:
"""Execute external provider search."""
"""Execute external plugin search."""
if not provider_name or not query:
if not plugin_name or not query:
from SYS import pipeline as ctx_mod
progress = None
if hasattr(ctx_mod, "get_pipeline_state"):
@@ -1476,10 +1472,10 @@ class search_file(Cmdlet):
except Exception:
pass
log("Error: search-file -provider requires both provider and query", file=sys.stderr)
log("Error: search-file -plugin requires both plugin and query", file=sys.stderr)
log(f"Usage: {self.usage}", file=sys.stderr)
providers_map = list_search_providers(config)
providers_map = list_search_plugins(config)
available = [n for n, a in providers_map.items() if a]
unconfigured = [n for n, a in providers_map.items() if not a]
@@ -1500,7 +1496,7 @@ class search_file(Cmdlet):
if hasattr(ctx_mod, "get_pipeline_state"):
progress = ctx_mod.get_pipeline_state().live_progress
provider = get_search_provider(provider_name, config)
provider = get_search_plugin(plugin_name, config)
if not provider:
if progress:
try:
@@ -1508,9 +1504,9 @@ class search_file(Cmdlet):
except Exception:
pass
show_provider_config_panel([provider_name])
show_provider_config_panel([plugin_name])
providers_map = list_search_providers(config)
providers_map = list_search_plugins(config)
available = [n for n, a in providers_map.items() if a]
if available:
show_available_providers_panel(available)
@@ -1522,7 +1518,7 @@ class search_file(Cmdlet):
worker_id,
"search-file",
title=f"Search: {query}",
description=f"Provider: {provider_name}, Query: {query}",
description=f"Plugin: {plugin_name}, Query: {query}",
)
except Exception:
pass
@@ -1532,7 +1528,7 @@ class search_file(Cmdlet):
from SYS.result_table import Table
provider_text = str(provider_name or "").strip()
provider_text = str(plugin_name or "").strip()
provider_lower = provider_text.lower()
# Dynamic query/filter extraction via provider
@@ -1564,9 +1560,9 @@ class search_file(Cmdlet):
source_cmd, source_args = provider.get_source_command(args_list)
table.set_source_command(source_cmd, source_args)
debug(f"[search-file] Calling {provider_name}.search(filters={search_filters})")
debug(f"[search-file] Calling {plugin_name}.search(filters={search_filters})")
results = provider.search(query, limit=limit, filters=search_filters or None)
debug(f"[search-file] {provider_name} -> {len(results or [])} result(s)")
debug(f"[search-file] {plugin_name} -> {len(results or [])} result(s)")
# Allow providers to apply provider-specific UX transforms (e.g. auto-expansion)
try:
@@ -1615,7 +1611,7 @@ class search_file(Cmdlet):
# Ensure provider source is present so downstream cmdlets (select) can resolve provider
if "source" not in item_dict:
item_dict["source"] = provider_name
item_dict["source"] = plugin_name
row_index = len(table.rows)
table.add_result(search_result)
@@ -1636,7 +1632,7 @@ class search_file(Cmdlet):
return 0
except Exception as exc:
log(f"Error searching provider '{provider_name}': {exc}", file=sys.stderr)
log(f"Error searching plugin '{plugin_name}': {exc}", file=sys.stderr)
import traceback
debug(traceback.format_exc())
@@ -1728,9 +1724,9 @@ class search_file(Cmdlet):
f.lower()
for f in (flag_registry.get("limit") or {"-limit", "--limit"})
}
provider_flags = {
plugin_flags = {
f.lower()
for f in (flag_registry.get("provider") or {"-provider", "--provider"})
for f in (flag_registry.get("plugin") or {"-plugin", "--plugin"})
}
open_flags = {
f.lower()
@@ -1740,7 +1736,7 @@ class search_file(Cmdlet):
# Parse arguments
query = ""
storage_backend: Optional[str] = None
provider_name: Optional[str] = None
plugin_name: Optional[str] = None
open_id: Optional[int] = None
limit = 100
limit_set = False
@@ -1756,8 +1752,8 @@ class search_file(Cmdlet):
query = f"{query} {chunk}".strip() if query else chunk
i += 2
continue
if low in provider_flags and i + 1 < len(args_list):
provider_name = args_list[i + 1]
if low in plugin_flags and i + 1 < len(args_list):
plugin_name = args_list[i + 1]
i += 2
continue
if low in open_flags and i + 1 < len(args_list):
@@ -1790,9 +1786,9 @@ class search_file(Cmdlet):
query = query.strip()
if provider_name:
return self._run_provider_search(
provider_name=provider_name,
if plugin_name:
return self._run_plugin_search(
plugin_name=plugin_name,
query=query,
limit=limit,
limit_set=limit_set,
@@ -1814,56 +1810,6 @@ class search_file(Cmdlet):
if store_filter and not storage_backend:
storage_backend = store_filter
# If the user accidentally used `-store <provider>` or `store:<provider>`,
# prefer to treat it as a provider search (providers like 'alldebrid' are not store backends).
try:
from Store.registry import list_configured_backend_names
providers_map = list_search_providers(config)
configured = list_configured_backend_names(config or {})
if storage_backend:
matched = None
storage_hint = self._normalize_lookup_target(storage_backend)
if storage_hint:
for p in (providers_map or {}):
if self._normalize_lookup_target(p) == storage_hint:
matched = p
break
if matched and str(storage_backend) not in configured:
log(f"Note: Treating '-store {storage_backend}' as provider search for '{matched}'", file=sys.stderr)
return self._run_provider_search(
provider_name=matched,
query=query,
limit=limit,
limit_set=limit_set,
open_id=open_id,
args_list=args_list,
refresh_mode=refresh_mode,
config=config,
)
elif store_filter:
matched = None
store_hint = self._normalize_lookup_target(store_filter)
if store_hint:
for p in (providers_map or {}):
if self._normalize_lookup_target(p) == store_hint:
matched = p
break
if matched and str(store_filter) not in configured:
log(f"Note: Treating 'store:{store_filter}' as provider search for '{matched}'", file=sys.stderr)
return self._run_provider_search(
provider_name=matched,
query=query,
limit=limit,
limit_set=limit_set,
open_id=open_id,
args_list=args_list,
refresh_mode=refresh_mode,
config=config,
)
except Exception:
# Be conservative: if provider detection fails, fall back to store behaviour
pass
hash_query = parse_hash_query(query)
web_plan = self._build_web_search_plan(
cmdnat/_status_shared.py
+57 -25
View File
@@ -74,34 +74,9 @@ def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
def provider_display_name(key: str) -> str:
label = (key or "").strip()
lower = label.lower()
if lower == "openlibrary":
return "OpenLibrary"
if lower == "alldebrid":
return "AllDebrid"
if lower == "youtube":
return "YouTube"
return label[:1].upper() + label[1:] if label else "Provider"
def default_provider_ping_targets(provider_key: str) -> list[str]:
provider = (provider_key or "").strip().lower()
if provider == "openlibrary":
return ["https://openlibrary.org"]
if provider == "youtube":
return ["https://www.youtube.com"]
if provider == "bandcamp":
return ["https://bandcamp.com"]
if provider == "libgen":
try:
from Provider.libgen import MIRRORS
return [str(url).rstrip("/") + "/json.php" for url in (MIRRORS or []) if str(url).strip()]
except ImportError:
return []
return []
def ping_first(urls: list[str]) -> tuple[bool, str]:
for url in urls:
ok, detail = ping_url(url)
@@ -110,3 +85,60 @@ def ping_first(urls: list[str]) -> tuple[bool, str]:
if urls:
return ping_url(urls[0])
return False, "No ping target"
def collect_plugin_startup_checks(config: dict) -> list[dict[str, Any]]:
provider_cfg = config.get("provider") if isinstance(config, dict) else None
if not isinstance(provider_cfg, dict) or not provider_cfg:
return []
try:
from ProviderCore.registry import get_plugin_class
except Exception:
return []
checks: list[dict[str, Any]] = []
for plugin_name in provider_cfg.keys():
plugin_key = str(plugin_name or "").strip().lower()
if not plugin_key:
continue
plugin_class = None
try:
plugin_class = get_plugin_class(plugin_key)
except Exception:
plugin_class = None
if plugin_class is None:
checks.append(
{
"status": "UNKNOWN",
"name": provider_display_name(plugin_key),
"plugin": plugin_key,
"detail": "Not registered",
}
)
continue
try:
plugin = plugin_class(config)
summary = plugin.status_summary()
except Exception as exc:
summary = {
"status": "DISABLED",
"name": provider_display_name(plugin_key),
"plugin": plugin_key,
"detail": str(exc),
}
checks.append(
{
"status": str(summary.get("status") or "UNKNOWN"),
"name": str(summary.get("name") or provider_display_name(plugin_key)),
"plugin": str(summary.get("plugin") or plugin_key),
"detail": str(summary.get("detail") or ""),
"files": summary.get("files"),
}
)
return checks
cmdnat/matrix.py
+49 -74
View File
@@ -15,6 +15,7 @@ from SYS.result_table import Table
from SYS.item_accessors import get_sha256_hex
from SYS.utils import extract_hydrus_hash_from_url
from SYS import pipeline as ctx
from ProviderCore.registry import get_plugin, get_plugin_for_url
from cmdnat._parsing import (
extract_arg_value,
extract_piped_value as _extract_piped_value,
@@ -29,6 +30,29 @@ _MATRIX_MENU_STATE_KEY = "matrix_menu_state"
_MATRIX_SELECTED_SETTING_KEY_KEY = "matrix_selected_setting_key"
def _get_matrix_provider(config: Dict[str, Any]) -> Any:
provider = get_plugin("matrix", config)
if provider is None:
raise RuntimeError("Matrix plugin is not registered")
return provider
def _resolve_plugin_url(url: str, config: Dict[str, Any]) -> str:
target = str(url or "").strip()
if not target:
return target
provider = get_plugin_for_url(target, config)
if provider is None:
return target
try:
resolved = provider.resolve_url(target)
except Exception:
return target
return str(resolved or target)
def _extract_set_value_arg(args: Sequence[str]) -> Optional[str]:
"""Extract the value from -set-value flag."""
return extract_arg_value(args, flags={"-set-value"})
@@ -212,35 +236,11 @@ def _resolve_room_identifier(value: str, config: Dict[str, Any]) -> Optional[str
conf_ids = _parse_config_room_filter_ids(config)
if conf_ids:
# Attempt to fetch names for the configured IDs
try:
from Provider.matrix import Matrix
# Avoid __init__ network failures by requiring homeserver+token to exist
block = config.get("provider", {}).get("matrix", {})
if block and block.get("homeserver") and block.get("access_token"):
try:
m = Matrix(config)
rooms = m.list_rooms(room_ids=conf_ids)
for room in rooms or []:
name = str(room.get("name") or "").strip()
rid = str(room.get("room_id") or "").strip()
if name and name.lower() == cand.lower():
return rid
if name and cand.lower() in name.lower():
return rid
except Exception:
# Best-effort; fallback below
pass
except Exception:
pass
# Last resort: attempt to ask the server for matching rooms (if possible)
try:
from Provider.matrix import Matrix
block = config.get("provider", {}).get("matrix", {})
if block and block.get("homeserver") and block.get("access_token"):
try:
m = Matrix(config)
rooms = m.list_rooms()
m = _get_matrix_provider(config)
rooms = m.list_rooms(room_ids=conf_ids)
for room in rooms or []:
name = str(room.get("name") or "").strip()
rid = str(room.get("room_id") or "").strip()
@@ -250,8 +250,22 @@ def _resolve_room_identifier(value: str, config: Dict[str, Any]) -> Optional[str
return rid
except Exception:
pass
except Exception:
pass
# Last resort: attempt to ask the server for matching rooms (if possible)
block = config.get("provider", {}).get("matrix", {})
if block and block.get("homeserver") and block.get("access_token"):
try:
m = _get_matrix_provider(config)
rooms = m.list_rooms()
for room in rooms or []:
name = str(room.get("name") or "").strip()
rid = str(room.get("room_id") or "").strip()
if name and name.lower() == cand.lower():
return rid
if name and cand.lower() in name.lower():
return rid
except Exception:
pass
return None
except Exception:
@@ -270,10 +284,8 @@ def _send_pending_to_rooms(config: Dict[str, Any], room_ids: List[str], args: Se
log("No pending items to upload (use: @N | .matrix)", file=sys.stderr)
return 1
from Provider.matrix import Matrix
try:
provider = Matrix(config)
provider = _get_matrix_provider(config)
except Exception as exc:
log(f"Matrix not available: {exc}", file=sys.stderr)
return 1
@@ -585,35 +597,6 @@ def _maybe_download_hydrus_file(item: Any,
return None
def _maybe_unlock_alldebrid_url(url: str, config: Dict[str, Any]) -> str:
try:
parsed = urlparse(url)
host = (parsed.netloc or "").lower()
if host != "alldebrid.com":
return url
if not (parsed.path or "").startswith("/f/"):
return url
try:
from Provider.alldebrid import _get_debrid_api_key # type: ignore
api_key = _get_debrid_api_key(config or {})
except Exception:
api_key = None
if not api_key:
return url
from API.alldebrid import AllDebridClient
client = AllDebridClient(str(api_key))
unlocked = client.unlock_link(url)
if isinstance(unlocked, str) and unlocked.strip():
return unlocked.strip()
except Exception:
pass
return url
def _resolve_upload_path(item: Any, config: Dict[str, Any]) -> Optional[str]:
"""Resolve a usable local file path for uploading.
@@ -645,7 +628,7 @@ def _resolve_upload_path(item: Any, config: Dict[str, Any]) -> Optional[str]:
return None
# Best-effort: unlock AllDebrid file links (they require auth and aren't directly downloadable).
url = _maybe_unlock_alldebrid_url(url, config)
url = _resolve_plugin_url(url, config)
try:
from API.HTTP import _download_direct_file
@@ -851,10 +834,8 @@ def _handle_settings_edit(result: Any, args: Sequence[str], config: Dict[str, An
def _handle_settings_test(config: Dict[str, Any]) -> int:
"""Test Matrix credentials and prompt for default rooms upon success."""
from Provider.matrix import Matrix
try:
provider = Matrix(config)
provider = _get_matrix_provider(config)
except Exception as exc:
log(f"Matrix test failed: {exc}", file=sys.stderr)
return 1
@@ -863,13 +844,11 @@ def _handle_settings_test(config: Dict[str, Any]) -> int:
return _show_default_room_picker(config, provider=provider)
def _show_default_room_picker(config: Dict[str, Any], *, provider: Optional["Matrix"] = None) -> int:
def _show_default_room_picker(config: Dict[str, Any], *, provider: Optional[Any] = None) -> int:
"""Display joined rooms so the user can select defaults for sharing."""
from Provider.matrix import Matrix
try:
if provider is None:
provider = Matrix(config)
provider = _get_matrix_provider(config)
except Exception as exc:
log(f"Matrix not available: {exc}", file=sys.stderr)
return 1
@@ -977,10 +956,8 @@ def _handle_settings_rooms(result: Any, args: Sequence[str], config: Dict[str, A
def _show_rooms_table(config: Dict[str, Any]) -> int:
"""Display rooms (refactored original behavior)."""
from Provider.matrix import Matrix
try:
provider = Matrix(config)
provider = _get_matrix_provider(config)
except Exception as exc:
log(f"Matrix not available: {exc}", file=sys.stderr)
return 1
@@ -1121,10 +1098,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
log("No pending items to upload (use: @N | .matrix)", file=sys.stderr)
return 1
from Provider.matrix import Matrix
try:
provider = Matrix(config)
provider = _get_matrix_provider(config)
except Exception as exc:
log(f"Matrix not available: {exc}", file=sys.stderr)
return 1
cmdnat/pipe.py
+75 -67
View File
@@ -10,17 +10,15 @@ from datetime import datetime, timedelta
from urllib.parse import urlparse, parse_qs
from pathlib import Path
from SYS.cmdlet_spec import Cmdlet, CmdletArg, parse_cmdlet_args
from Provider.tidal_manifest import resolve_tidal_manifest_path
from ProviderCore.registry import get_plugin_for_url
from SYS.logger import debug, get_thread_stream, is_debug_enabled, set_debug, set_thread_stream
from SYS.result_table import Table
from MPV.mpv_ipc import MPV
from SYS import pipeline as ctx
from SYS.models import PipeObject
from SYS.config import get_hydrus_access_key, get_hydrus_url
from SYS.config import get_hydrus_access_key, get_hydrus_url, resolve_cookies_path
_ALLDEBRID_UNLOCK_CACHE: Dict[str,
str] = {}
_NOTES_PREFETCH_INFLIGHT: set[str] = set()
_NOTES_PREFETCH_LOCK = threading.Lock()
_PLAYLIST_STORE_CACHE: Optional[Dict[str, Any]] = None
@@ -478,73 +476,85 @@ def _try_enable_mpv_file_logging(mpv_log_path: str, *, attempts: int = 3) -> boo
return bool(ok)
def _get_alldebrid_api_key(config: Optional[Dict[str, Any]]) -> Optional[str]:
def _iter_plugin_targets(item: Any) -> List[str]:
values: List[str] = []
seen: set[str] = set()
def _add(candidate: Any) -> None:
text = str(candidate or "").strip()
if not text or text in seen:
return
seen.add(text)
values.append(text)
try:
if not isinstance(config, dict):
return None
provider_cfg = config.get("provider")
if not isinstance(provider_cfg, dict):
return None
ad_cfg = provider_cfg.get("alldebrid")
if not isinstance(ad_cfg, dict):
return None
key = ad_cfg.get("api_key")
if not isinstance(key, str):
return None
key = key.strip()
return key or None
if isinstance(item, dict):
_add(item.get("path"))
_add(item.get("url"))
_add(item.get("source_url"))
_add(item.get("target"))
metadata = item.get("full_metadata") or item.get("metadata")
else:
_add(getattr(item, "path", None))
_add(getattr(item, "url", None))
_add(getattr(item, "source_url", None))
_add(getattr(item, "target", None))
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if isinstance(metadata, dict):
_add(metadata.get("url"))
_add(metadata.get("webpage_url"))
_add(metadata.get("source_url"))
extra = item.get("extra") if isinstance(item, dict) else getattr(item, "extra", None)
if isinstance(extra, dict):
_add(extra.get("url"))
_add(extra.get("source_url"))
except Exception:
return None
return values
return values
def _is_alldebrid_protected_url(url: str) -> bool:
def _resolve_plugin_url(url: str, config: Optional[Dict[str, Any]]) -> str:
target = str(url or "").strip()
if not target:
return target
try:
if not isinstance(url, str):
return False
u = url.strip()
if not u.startswith(("http://", "https://")):
return False
p = urlparse(u)
host = (p.netloc or "").lower()
path = p.path or ""
# AllDebrid file page links (require auth; not directly streamable by mpv)
return host == "alldebrid.com" and path.startswith("/f/")
plugin = get_plugin_for_url(target, config or {})
except Exception:
return False
def _maybe_unlock_alldebrid_url(url: str, config: Optional[Dict[str, Any]]) -> str:
"""Convert AllDebrid protected file URLs into direct streamable links.
When AllDebrid returns `https://alldebrid.com/f/...`, that URL typically requires
authentication. MPV cannot access it without credentials. We transparently call
the AllDebrid API `link/unlock` (using the configured API key) to obtain a direct
URL that MPV can stream.
"""
if not _is_alldebrid_protected_url(url):
return url
cached = _ALLDEBRID_UNLOCK_CACHE.get(url)
if isinstance(cached, str) and cached:
return cached
api_key = _get_alldebrid_api_key(config)
if not api_key:
return url
plugin = None
if plugin is None:
return target
try:
from API.alldebrid import AllDebridClient
resolved = plugin.resolve_url(target)
except Exception as exc:
debug(f"Plugin URL resolution failed for {target}: {exc}", file=sys.stderr)
return target
client = AllDebridClient(api_key)
unlocked = client.unlock_link(url)
if isinstance(unlocked, str) and unlocked.strip():
unlocked = unlocked.strip()
_ALLDEBRID_UNLOCK_CACHE[url] = unlocked
return unlocked
except Exception as e:
debug(f"AllDebrid unlock failed for MPV target: {e}", file=sys.stderr)
return str(resolved or target)
return url
def _resolve_plugin_playback_path(item: Any, config: Optional[Dict[str, Any]]) -> Optional[str]:
for candidate in _iter_plugin_targets(item):
try:
plugin = get_plugin_for_url(candidate, config or {})
except Exception:
plugin = None
if plugin is None:
continue
try:
resolved = plugin.resolve_playback_path(item)
except Exception as exc:
debug(f"Plugin playback-path resolution failed for {candidate}: {exc}", file=sys.stderr)
continue
text = str(resolved or "").strip()
if text:
return text
return None
def _ensure_lyric_overlay(mpv: MPV) -> None:
@@ -1078,9 +1088,7 @@ def _ensure_ytdl_cookies(config: Optional[Dict[str, Any]] = None) -> None:
cookies_path = None
try:
from tool.ytdlp import YtDlpTool
cookiefile = YtDlpTool(config or {}).resolve_cookiefile()
cookiefile = resolve_cookies_path(config or {})
if cookiefile is not None:
cookies_path = str(cookiefile)
except Exception:
@@ -1326,7 +1334,7 @@ def _get_playable_path(
"none"}:
path = None
manifest_path = resolve_tidal_manifest_path(item)
manifest_path = _resolve_plugin_playback_path(item, config)
if manifest_path:
path = manifest_path
else:
@@ -1542,7 +1550,7 @@ def _queue_items(
# If the target is an AllDebrid protected file URL, unlock it to a direct link for MPV.
try:
if isinstance(target, str):
target = _maybe_unlock_alldebrid_url(target, config)
target = _resolve_plugin_url(target, config)
except Exception:
pass
@@ -2591,7 +2599,7 @@ def _start_mpv(
try:
needs_mpd_whitelist = False
for it in items or []:
mpd = resolve_tidal_manifest_path(it)
mpd = _resolve_plugin_playback_path(it, config)
candidate = mpd
if not candidate:
if isinstance(it, dict):
cmdnat/status.py
+12 -79
View File
@@ -4,18 +4,14 @@ import shutil
from typing import Any, Dict, List
from SYS.cmdlet_spec import Cmdlet
from SYS.config import resolve_cookies_path
from SYS import pipeline as ctx
from SYS.result_table import Table
from SYS.logger import set_debug, debug
from cmdnat._status_shared import (
add_startup_check as _add_startup_check,
default_provider_ping_targets as _default_provider_ping_targets,
has_provider as _has_provider,
collect_plugin_startup_checks as _collect_plugin_startup_checks,
has_store_subtype as _has_store_subtype,
has_tool as _has_tool,
ping_first as _ping_first,
ping_url as _ping_url,
provider_display_name as _provider_display_name,
)
CMDLET = Cmdlet(
@@ -95,82 +91,19 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int:
detail = f"{uval} - {err or 'Unavailable'}"
_add_startup_check(startup_table, status, nkey, store="hydrusnetwork", files=files, detail=detail)
# Providers
pcfg = config.get("provider", {})
if isinstance(pcfg, dict) and pcfg:
from ProviderCore.registry import list_providers, list_search_providers, list_file_providers
from Provider.metadata_provider import list_metadata_providers
p_avail = list_providers(config) or {}
s_avail = list_search_providers(config) or {}
f_avail = list_file_providers(config) or {}
m_avail = list_metadata_providers(config) or {}
debug(f"Provider registries: providers={list(p_avail.keys())}, search={list(s_avail.keys())}, file={list(f_avail.keys())}, metadata={list(m_avail.keys())}")
already = {"matrix"}
for pname in pcfg.keys():
prov = str(pname).lower()
if prov in already: continue
display = _provider_display_name(prov)
if prov == "alldebrid":
try:
from Provider.alldebrid import _get_debrid_api_key
from API.alldebrid import AllDebridClient
api_key = _get_debrid_api_key(config)
debug(f"AllDebrid configured: api_key_present={bool(api_key)}")
if not api_key:
_add_startup_check(startup_table, "DISABLED", display, provider=prov, detail="Not configured")
else:
client = AllDebridClient(api_key)
_add_startup_check(startup_table, "ENABLED", display, provider=prov, detail=getattr(client, "base_url", "Connected"))
debug(f"AllDebrid client connected: base_url={getattr(client, 'base_url', 'unknown')}")
except Exception as exc:
_add_startup_check(startup_table, "DISABLED", display, provider=prov, detail=str(exc))
debug(f"AllDebrid check failed: {exc}")
already.add(prov)
continue
is_known = prov in p_avail or prov in s_avail or prov in f_avail or prov in m_avail
if not is_known:
_add_startup_check(startup_table, "UNKNOWN", display, provider=prov, detail="Not registered")
debug(f"Provider {prov} not registered")
else:
ok_val = p_avail.get(prov) or s_avail.get(prov) or f_avail.get(prov) or m_avail.get(prov)
detail = "Configured" if ok_val else "Not configured"
ping_targets = _default_provider_ping_targets(prov)
if ping_targets:
debug(f"Provider {prov} ping targets: {ping_targets}")
pok, pdet = _ping_first(ping_targets)
debug(f"Provider {prov} ping result: ok={pok}, detail={pdet}")
detail = pdet if ok_val else f"{detail} | {pdet}"
_add_startup_check(startup_table, "ENABLED" if ok_val else "DISABLED", display, provider=prov, detail=detail)
already.add(prov)
# Matrix
if _has_provider(config, "matrix"):
try:
from Provider.matrix import Matrix
m_prov = Matrix(config)
mcfg = config.get("provider", {}).get("matrix", {})
hs = str(mcfg.get("homeserver") or "").strip()
rid = str(mcfg.get("room_id") or "").strip()
detail = f"{hs} room:{rid}"
valid = False
try:
valid = bool(m_prov.validate())
except Exception as exc:
debug(f"Matrix validate failed: {exc}")
_add_startup_check(startup_table, "ENABLED" if valid else "DISABLED", "Matrix", provider="matrix", detail=detail)
debug(f"Matrix check: homeserver={hs}, room_id={rid}, validate={valid}")
except Exception as exc:
_add_startup_check(startup_table, "DISABLED", "Matrix", provider="matrix", detail=str(exc))
debug(f"Matrix instantiation failed: {exc}")
for check in _collect_plugin_startup_checks(config):
_add_startup_check(
startup_table,
str(check.get("status") or "UNKNOWN"),
str(check.get("name") or "Plugin"),
provider=str(check.get("plugin") or ""),
files=check.get("files"),
detail=str(check.get("detail") or ""),
)
# Cookies
try:
from tool.ytdlp import YtDlpTool
cf = YtDlpTool(config).resolve_cookiefile()
cf = resolve_cookies_path(config)
_add_startup_check(startup_table, "FOUND" if cf else "MISSING", "Cookies", detail=str(cf) if cf else "Not found")
debug(f"Cookies: resolved cookiefile={cf}")
except Exception as exc:
cmdnat/telegram.py
+9 -3
View File
@@ -8,10 +8,18 @@ from SYS.cmdlet_spec import Cmdlet, CmdletArg
from SYS.logger import log
from SYS.result_table import Table
from SYS import pipeline as ctx
from ProviderCore.registry import get_plugin
from cmdnat._parsing import has_flag as _has_flag, normalize_to_list as _normalize_to_list
_TELEGRAM_PENDING_ITEMS_KEY = "telegram_pending_items"
def _get_telegram_provider(config: Dict[str, Any]) -> Any:
provider = get_plugin("telegram", config)
if provider is None:
raise RuntimeError("Telegram plugin is not registered")
return provider
def _extract_chat_id(chat_obj: Any) -> Optional[int]:
try:
if isinstance(chat_obj, dict):
@@ -119,10 +127,8 @@ def _extract_file_path(item: Any) -> Optional[str]:
def _run(_result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
from Provider.telegram import Telegram
try:
provider = Telegram(config)
provider = _get_telegram_provider(config)
except Exception as exc:
log(f"Telegram not available: {exc}", file=sys.stderr)
return 1
docs/provider_authoring.md
+12 -12
View File
@@ -1,13 +1,13 @@
# Provider authoring: ResultTable & provider adapters ✅
# Plugin authoring: ResultTable & plugin adapters
This short guide explains how to write providers that integrate with the *strict* ResultTable API: adapters must yield `ResultModel` instances and providers register via `SYS.result_table_adapters.register_provider` with a column specification and a `selection_fn`.
This short guide explains how to write plugins that integrate with the *strict* ResultTable API: adapters must yield `ResultModel` instances and plugins register via `SYS.result_table_adapters.register_plugin` with a column specification and a `selection_fn`.
---
## Quick summary
- Providers register a *provider adapter* (callable that yields `ResultModel`).
- Providers must also provide `columns` (static list or factory) and a `selection_fn` that returns CLI args for a selected row.
- Plugins register a *plugin adapter* (callable that yields `ResultModel`).
- Plugins must also provide `columns` (static list or factory) and a `selection_fn` that returns CLI args for a selected row.
- For simple HTML table/list scraping, prefer `TableProviderMixin` from `SYS.provider_helpers` to fetch and extract rows using `SYS.html_table.extract_records`.
## Runtime dependency policy
@@ -21,11 +21,11 @@ This short guide explains how to write providers that integrate with the *strict
## Minimal provider template (copy/paste)
```py
# Provider/my_provider.py
# plugins/my_plugin.py
from typing import Any, Dict, Iterable, List
from SYS.result_table_api import ResultModel, ColumnSpec, title_column, metadata_column
from SYS.result_table_adapters import register_provider
from SYS.result_table_adapters import register_plugin
# Example adapter: convert provider-specific items into ResultModel instances
SAMPLE_ITEMS = [
@@ -59,8 +59,8 @@ def selection_fn(row: ResultModel) -> List[str]:
return ["-path", row.path]
return ["-title", row.title or ""]
# Register provider (done at import time)
register_provider("myprovider", adapter, columns=columns_factory, selection_fn=selection_fn)
# Register plugin (done at import time)
register_plugin("myprovider", adapter, columns=columns_factory, selection_fn=selection_fn)
```
---
@@ -84,7 +84,7 @@ class MyTableProvider(TableProviderMixin, Provider):
return self.search_table_from_url(url, limit=limit)
```
`TableProviderMixin.search_table_from_url` returns `ProviderCore.base.SearchResult` entries. If you want to integrate this provider with the strict `ResultTable` registry, add a small adapter that converts `SearchResult` -> `ResultModel` and register it using `register_provider` (see `Provider/vimm.py` for a real example).
`TableProviderMixin.search_table_from_url` returns `ProviderCore.base.SearchResult` entries. If you want to integrate this plugin with the strict `ResultTable` registry, add a small adapter that converts `SearchResult` -> `ResultModel` and register it using `register_plugin` (see `Provider/vimm.py` for a real example).
---
@@ -93,7 +93,7 @@ class MyTableProvider(TableProviderMixin, Provider):
- `columns` may be a static `List[ColumnSpec]` or a factory `def cols(rows: List[ResultModel]) -> List[ColumnSpec]` that inspects sample rows.
- `selection_fn` must accept a `ResultModel` and return a `List[str]` representing CLI args (e.g., `['-path', row.path]`). These args are used by `select` and `@N` expansion.
**Tip:** for providers that produce downloadable file rows prefer returning explicit URL args (e.g., `['-url', row.path]`) so the selected URL is clearly identified by downstream downloaders and to avoid ambiguous parsing when provider hints (like `-provider`) are present.
**Tip:** for plugins that produce downloadable file rows prefer returning explicit URL args (e.g., `['-url', row.path]`) so the selected URL is clearly identified by downstream downloaders and to avoid ambiguous parsing when plugin hints (like `-plugin`) are present.
- Ensure your `ResultModel.source` is set (either in the model or rely on the provider name set by `serialize_row`).
---
@@ -107,7 +107,7 @@ class MyTableProvider(TableProviderMixin, Provider):
## Testing & examples
- Write `tests/test_provider_<name>.py` that imports your provider and verifies `provider.build_table(...)` produces a `ResultTable` (has `.rows` and `.columns`) and that `serialize_rows()` yields dicts with `_selection_args`, `_selection_action` when applicable, and `source`.
- When you need to guarantee a specific CLI stage sequence (e.g., `download-file -url <path> -provider <name>`), call `table.set_row_selection_action(row_index, tokens)` so the serialized payload emits `_selection_action` and the CLI can run the row exactly as intended.
- When you need to guarantee a specific CLI stage sequence (e.g., `download-file -url <path> -plugin <name>`), call `table.set_row_selection_action(row_index, tokens)` so the serialized payload emits `_selection_action` and the CLI can run the row exactly as intended.
- For table providers you can test `search_table_from_url` using a local HTML fixture or by mocking `HTTPClient` to return a small sample page.
- If you rely on pandas, add a test that monkeypatches `sys.modules['pandas']` to a simple shim to validate the pandas path.
@@ -119,7 +119,7 @@ from Provider import example_provider
def test_example_provider_registration():
provider = get_provider("example")
plugin = get_plugin("example")
rows = list(provider.adapter(example_provider.SAMPLE_ITEMS))
assert rows and rows[0].title
cols = provider.get_columns(rows)
docs/provider_guide.md
+20 -21
View File
@@ -1,23 +1,23 @@
# Provider Development Guide
# Plugin Development Guide
## 🎯 Purpose
This guide describes how to write, test, and register a provider so the application can discover and use it as a pluggable component.
This guide describes how to write, test, and register a plugin so the application can discover and use it as a pluggable component.
> Keep provider code small, focused, and well-tested. Use existing providers as examples.
> Keep plugin code small, focused, and well-tested. Built-in plugins live in `Provider/` and external drop-in plugins live under `plugins/`.
---
## 🔧 Anatomy of a Provider
A provider is a Python class that extends `ProviderCore.base.Provider` and implements a few key methods and attributes.
## 🔧 Anatomy of a Plugin
A plugin is a Python class that extends `ProviderCore.base.Provider` and implements a few key methods and attributes.
Minimum expectations:
- `class MyProvider(Provider):` — subclass the base provider
- `class MyPlugin(Provider):` — subclass the base plugin class
- `URL` / `URL_DOMAINS` or `url_patterns()` — to let the registry route URLs
- `validate(self) -> bool` — return True when provider is configured and usable
- `validate(self) -> bool` — return True when the plugin is configured and usable
- `search(self, query, limit=50, filters=None, **kwargs)` — return a list of `SearchResult`
Optional but common:
- `download(self, result: SearchResult, output_dir: Path) -> Optional[Path]` — download a provider result
- `download(self, result: SearchResult, output_dir: Path) -> Optional[Path]` — download a plugin result
- `selector(self, selected_items, *, ctx, stage_is_last=True, **kwargs) -> bool` — handle `@N` selections
- `download_url(self, url, output_dir, progress_cb=None)` — direct URL-handling helper
@@ -71,8 +71,8 @@ class HelloProvider(Provider):
---
## ⬇️ Implementing download() and download_url()
- Prefer provider `download(self, result, output_dir)` for piped provider items.
- For provider-provided URLs, implement `download_url` to allow `download-file` to route downloads through providers.
- Prefer plugin `download(self, result, output_dir)` for piped plugin items.
- For plugin-provided URLs, implement `download_url` to allow `download-file` to route downloads through plugins.
- Use the repo `_download_direct_file` helper for HTTP downloads when possible.
Example download():
@@ -90,12 +90,12 @@ def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
---
## 🧭 URL routing
Providers can declare:
Plugins can declare:
- `URL = ("magnet:",)` or similar prefix list
- `URL_DOMAINS = ("example.com",)` to match hosts
- Or override `@classmethod def url_patterns(cls):` to combine static and dynamic patterns
The registry uses these to match `download-file <url>` or to pick which provider should handle the URL.
The registry uses these to match `download-file <url>` or to pick which plugin should handle the URL.
---
@@ -106,8 +106,8 @@ The registry uses these to match `download-file <url>` or to pick which provider
---
## 🧪 Testing providers
- Keep tests small and local. Create `tests/test_provider_<name>.py`.
## 🧪 Testing plugins
- Keep tests small and local. Create `tests/test_provider_<name>.py` or another tracked test target.
- Test `search()` with mock HTTP responses (use `requests-mock` or similar).
- Test `download()` using a temp directory and a small file server or by mocking `_download_direct_file`.
- Test `selector()` by constructing a fake result and `ctx` object.
@@ -125,10 +125,9 @@ pytest -q
---
## 📦 Registration & packaging
- Add your provider module under `Provider/` and ensure it is imported by module package initialization. Common approach:
- Place file `Provider/myprovider.py`
- Ensure `Provider/__init__.py` imports the module (or the registry auto-discovers by package import)
- If the project has a central provider registry, add lookup helpers there (e.g., `ProviderCore/registry.py`). Usually providers register themselves at import time.
- Built-in plugins live under `Provider/` and are auto-discovered from that package.
- External user plugins can be dropped into `plugins/` or any directory listed in `MM_PLUGIN_PATH` / `MEDEIA_PLUGIN_PATH`.
- Plugin authors should import from `ProviderCore.*`.
---
@@ -147,19 +146,19 @@ pytest -q
- [ ] Provide `URL` / `URL_DOMAINS` or `url_patterns()` for routing
- [ ] Add `download()` or `download_url()` for piped/passed URL downloads
- [ ] Add tests under `tests/`
- [ ] Add module to `Provider/` package and ensure import/registration
- [ ] Add the plugin module to `Provider/` for built-ins, or drop it into `plugins/` for plug-and-play user installs
---
## 🔗 Further reading
- See existing providers in `Provider/` for patterns and edge cases.
- See existing built-in plugins in `Provider/` for patterns and edge cases.
- Check `API/` helpers for HTTP and debrid clients.
---
If you'd like, I can:
- Add an example provider file under `Provider/` as a template (see `Provider/hello_provider.py`), and
- Add an example plugin file under `Provider/` as a template (see `Provider/hello_provider.py`), and
- Create unit tests for it (see `tests/test_provider_hello.py`).
I have added a minimal example provider and tests in this repository; use them as a starting point for new providers.
docs/result_table.md
+6 -6
View File
@@ -40,7 +40,7 @@ from SYS.result_table import ResultTable
table = ResultTable("Provider: X result").set_preserve_order(True)
table.set_table("provider_name")
table.set_table_metadata({"provider":"provider_name","view":"folders"})
table.set_source_command("search-file", ["-provider","provider_name","query"])
table.set_source_command("search-file", ["-plugin","provider_name","query"])
for r in results:
table.add_result(r) # r can be a SearchResult, dict, or PipeObject
@@ -82,13 +82,13 @@ Example commands:
```
# List magnets in your account
search-file -provider alldebrid "*"
search-file -plugin alldebrid "*"
# Open magnet id 123 and list its files
search-file -provider alldebrid -open 123 "*"
search-file -plugin alldebrid -open 123 "*"
# Or expand via @ selection (selector handles drilling):
search-file -provider alldebrid "*"
search-file -plugin alldebrid "*"
@3 # selector will open the magnet referenced by row #3 and show the file table
```
@@ -147,7 +147,7 @@ Selection & download flows
```
# Expand magnet and add first file to local directory
search-file -provider alldebrid "*"
search-file -plugin alldebrid "*"
@3 # view files
@1 | add-file -path C:\mydir
```
@@ -167,7 +167,7 @@ Example usage:
```
# Search for an artist
search-file -provider bandcamp "artist:radiohead"
search-file -plugin bandcamp "artist:radiohead"
# Select an artist row to expand into releases
@1
docs/result_table_selector.md
+7 -7
View File
@@ -1,22 +1,22 @@
Selector & provider-table usage
Selector & plugin-table usage
This project provides a small provider/table/selector flow that allows providers
This project provides a small plugin/table/selector flow that allows plugins
and cmdlets to interact via a simple, pipable API.
Key ideas
- `provider-table` renders a provider result set and *emits* pipeline-friendly dicts for each row. Each emitted item includes `_selection_args`, a list of args the provider suggests for selecting that row (e.g., `['-path', '/tmp/file']`).
- `plugin-table` renders a plugin result set and *emits* pipeline-friendly dicts for each row. Each emitted item includes `_selection_args`, a list of args the plugin suggests for selecting that row (e.g., `['-path', '/tmp/file']`).
- Use the `@N` syntax to select an item from a table and chain it to the next cmdlet.
Example:
provider-table -provider example -sample | @1 | add-file -store default
plugin-table -plugin example -sample | @1 | add-file -store default
What providers must implement
What plugins must implement
- An adapter that yields `ResultModel` objects (breaking API).
- Optionally supply a `columns` factory and `selection_fn` (see `Provider/example_provider.py`).
Implementation notes
- `provider-table` emits dicts like `{ 'title': ..., 'path': ..., 'metadata': ..., '_selection_args': [...] }`.
- Selection syntax (`@1`) will prefer `_selection_args` if present; otherwise it will fall back to provider selection logic or sensible defaults (`-path` or `-title`).
- `plugin-table` emits dicts like `{ 'title': ..., 'path': ..., 'metadata': ..., '_selection_args': [...] }`.
- Selection syntax (`@1`) will prefer `_selection_args` if present; otherwise it will fall back to plugin selection logic or sensible defaults (`-path` or `-title`).
This design keeps the selector-focused UX small and predictable while enabling full cmdlet interoperability via piping and `-run-cmd`.
plugins/README.md
+39
View File
@@ -0,0 +1,39 @@
# External Plugins
Drop user plugins in this folder to make them available to the app without editing the built-in `Provider/` package.
Supported discovery paths:
- `plugins/` in the repo root
- `plugins/` in the current working directory
- Any directory listed in `MM_PLUGIN_PATH`
- Any directory listed in `MEDEIA_PLUGIN_PATH`
Plugin module rules:
- A plugin can be a single `.py` file or a package directory with `__init__.py`.
- Define a class that inherits from `ProviderCore.base.Provider`.
- Give it a stable name using `PLUGIN_NAME` or the class name.
Example skeleton:
```python
from ProviderCore.base import Provider, SearchResult
class MyPlugin(Provider):
PLUGIN_NAME = "myplugin"
URL_DOMAINS = ("example.com",)
def search(self, query, limit=50, filters=None, **kwargs):
text = str(query or "").strip()
if not text:
return []
return [
SearchResult(
table="myplugin",
title=f"Result for {text}",
path=f"https://example.com/{text}",
)
]
```
Built-in plugins still live in `Provider/`.
readme.md
+2 -2
View File
@@ -33,9 +33,9 @@ Medios-Macina is a API driven file media manager and virtual toolbox capable of
<li><i>no opening of folders neccessary! You can add multiple tags to a file and use the search engine to immediately find and retrieve that file your looking for</i></li>
<li><b>Flexible syntax structure:</b> chain commands with `|` and select options from tables with `@N`.</li>
<li><b>Multiple file stores:</b> *HYDRUSNETWORK*
- **Provider plugin integration:** *YOUTUBE, OPENLIBRARY, INTERNETARCHIVE, SOULSEEK, LIBGEN, ALLDEBRID, TELEGRAM, BANDCAMP*</li>
- **Plugin integration:** *YOUTUBE, OPENLIBRARY, INTERNETARCHIVE, SOULSEEK, LIBGEN, ALLDEBRID, TELEGRAM, BANDCAMP*</li>
<li><b>Module Mixing:</b> *[Playwright](https://github.com/microsoft/playwright), [yt-dlp](https://github.com/yt-dlp/yt-dlp), [typer](https://github.com/fastapi/typer)*</li>
<li><b>Optional stacks:</b> Telethon (Telegram), aioslsk (Soulseek), and the FlorenceVision tooling install automatically when you configure the corresponding provider/tool blocks.
<li><b>Optional stacks:</b> Telethon (Telegram), aioslsk (Soulseek), and the FlorenceVision tooling install automatically when you configure the corresponding plugin/tool blocks.
<li><b>MPV Manager:</b> Play audio, video, and even images in a custom designed MPV with trimming, screenshotting, and more built right in!</li>
<li><i>Supports remote access and networked setups for offsite servers and sharing workflows.</i></li>
</ul>