goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updating and refining plugin system refactor

Browse Source
This commit is contained in:
Nose
2026-04-28 22:20:54 -07:00
parent 8685fbb723
commit 323c24f4f4
33 changed files with 4287 additions and 3312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdlet/_shared.py
+10 -2
View File
@@ -190,10 +190,18 @@ class SharedArgs:
name="store",
type="enum",
choices=[], # Dynamically populated via get_store_choices()
description="Selects store",
description="Selects a storage backend",
query_key="store",
)
INSTANCE = CmdletArg(
name="instance",
type="string",
description="Selects a plugin instance",
query_key="instance",
query_aliases=["store"],
)
URL = CmdletArg(
name="url",
type="string",
@@ -1410,7 +1418,7 @@ def fetch_hydrus_metadata(
Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
Args:
config: Configuration object used to resolve the Hydrus provider/store
config: Configuration object used to resolve the Hydrus plugin/store
hash_hex: File hash to fetch metadata for
store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
cmdlet/add_file.py
+42 -9
View File
@@ -15,6 +15,7 @@ from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.rich_display import show_available_plugins_panel, show_plugin_config_panel
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store
from API.HTTP import _download_direct_file
@@ -178,10 +179,11 @@ class Add_File(Cmdlet):
summary=
"Ingest a local media file to a store backend, upload plugin, or local directory.",
usage=
"add-file (-path <filepath> | <piped>) (-storage <location> | -plugin <upload-plugin>) [-delete]",
"add-file (-path <filepath> | <piped>) (-store <backend|path> | -plugin <upload-plugin>) [-instance NAME] [-delete]",
arg=[
SharedArgs.PATH,
SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.URL,
SharedArgs.PLUGIN,
CmdletArg(
@@ -194,7 +196,7 @@ class Add_File(Cmdlet):
],
detail=[
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
"- Storage location options (use -storage):",
"- Storage location options (use -store):",
" hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory",
" <path>: Copy file to specified directory",
@@ -202,9 +204,12 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- In plugin mode, -store <name> is still accepted as a compatibility alias for -instance <name>.",
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -store tutorial',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
],
exec=self.run,
)
@@ -223,9 +228,12 @@ class Add_File(Cmdlet):
path_arg = parsed.get("path")
location = parsed.get("store")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url")
plugin_name = parsed.get("plugin")
delete_after = parsed.get("delete", False)
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Convenience: when piping a file into add-file, allow `-path <existing dir>`
# to act as the destination export directory.
@@ -412,6 +420,7 @@ class Add_File(Cmdlet):
("items", total_items),
("location", location),
("plugin", plugin_name),
("instance", plugin_instance),
("delete", delete_after),
],
border_style="cyan",
@@ -647,6 +656,7 @@ class Add_File(Cmdlet):
code = self._handle_plugin_upload(
media_path,
plugin_name,
plugin_instance,
pipe_obj,
config,
delete_after_item
@@ -1442,9 +1452,9 @@ class Add_File(Cmdlet):
if not plugin_key:
return None, None, None
from ProviderCore.registry import get_search_plugin
from ProviderCore.registry import get_plugin
plugin = get_search_plugin(plugin_key, config)
plugin = get_plugin(plugin_key, config)
if plugin is None:
return None, None, None
@@ -1762,6 +1772,7 @@ class Add_File(Cmdlet):
*,
hash_value: str,
store: str,
provider: Optional[str] = None,
path: Optional[str],
tag: List[str],
title: Optional[str],
@@ -1770,6 +1781,7 @@ class Add_File(Cmdlet):
) -> None:
pipe_obj.hash = hash_value
pipe_obj.store = store
pipe_obj.provider = provider
pipe_obj.is_temp = False
pipe_obj.path = path
pipe_obj.tag = tag
@@ -2180,23 +2192,42 @@ class Add_File(Cmdlet):
def _handle_plugin_upload(
media_path: Path,
plugin_name: str,
instance_name: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
delete_after: bool,
) -> int:
"""Handle uploading via an upload plugin (e.g. 0x0)."""
from ProviderCore.registry import get_upload_plugin
from ProviderCore.registry import (
get_plugin_with_capability,
list_plugin_names_with_capability,
list_plugins_with_capability,
)
log(f"Uploading via {plugin_name}: {media_path.name}", file=sys.stderr)
try:
file_provider = get_upload_plugin(plugin_name, config)
file_provider = get_plugin_with_capability(plugin_name, "upload", config)
if not file_provider:
log(f"Upload plugin '{plugin_name}' not available", file=sys.stderr)
available_map = list_plugins_with_capability("upload", config)
known_upload_plugins = set(list_plugin_names_with_capability("upload"))
available_uploads = [name for name, enabled in available_map.items() if enabled and name in known_upload_plugins]
if str(plugin_name or "").strip().lower() in known_upload_plugins:
show_plugin_config_panel([plugin_name])
else:
log(f"Upload plugin '{plugin_name}' is not available or does not support upload", file=sys.stderr)
if available_uploads:
show_available_plugins_panel(sorted(available_uploads))
return 1
hoster_url = file_provider.upload(str(media_path), pipe_obj=pipe_obj)
hoster_url = file_provider.upload(
str(media_path),
pipe_obj=pipe_obj,
instance=instance_name,
)
log(f"File uploaded: {hoster_url}", file=sys.stderr)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
@@ -2209,6 +2240,7 @@ class Add_File(Cmdlet):
extra_updates: Dict[str,
Any] = {
"plugin": plugin_name,
"instance": instance_name,
"plugin_url": hoster_url,
}
if isinstance(pipe_obj.extra, dict):
@@ -2222,7 +2254,8 @@ class Add_File(Cmdlet):
Add_File._update_pipe_object_destination(
pipe_obj,
hash_value=f_hash or "unknown",
store=plugin_name or "plugin",
store="",
provider=plugin_name or None,
path=file_path,
tag=pipe_obj.tag,
title=pipe_obj.title or (media_path.name if media_path else None),
cmdlet/download_file.py
+13 -13
View File
@@ -55,12 +55,13 @@ class Download_File(Cmdlet):
name="download-file",
summary="Download files or streaming media",
usage=
"download-file <url> [-path DIR] [options] OR @N | download-file [-path DIR|DIR] [options]",
"download-file <url> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]",
alias=["dl-file",
"download-http"],
arg=[
SharedArgs.URL,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
SharedArgs.QUERY,
QueryArg(
@@ -85,6 +86,7 @@ class Download_File(Cmdlet):
],
detail=[
"Download files directly via HTTP or streaming media via yt-dlp.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
],
exec=self.run,
@@ -522,13 +524,13 @@ class Download_File(Cmdlet):
config: Dict[str,
Any],
) -> List[Any]:
get_search_plugin = registry.get("get_search_plugin")
get_provider = registry.get("get_plugin")
expanded_items: List[Any] = []
for item in piped_items:
try:
provider_key = self._provider_key_from_item(item)
provider = get_search_plugin(provider_key, config) if provider_key and get_search_plugin else None
provider = get_provider(provider_key, config) if provider_key and get_provider else None
# Generic hook: If provider has expand_item(item), use it.
if provider and hasattr(provider, "expand_item") and callable(provider.expand_item):
@@ -566,7 +568,7 @@ class Download_File(Cmdlet):
) -> tuple[int, int]:
downloaded_count = 0
queued_magnet_submissions = 0
get_search_plugin = registry.get("get_search_plugin")
get_provider = registry.get("get_plugin")
SearchResult = registry.get("SearchResult")
expanded_items = self._expand_provider_items(
@@ -622,15 +624,15 @@ class Download_File(Cmdlet):
transfer_label = label
# If this looks like a provider item and providers are available, prefer provider.download()
# If this looks like a plugin-owned item and a plugin is available, prefer plugin.download().
downloaded_path: Optional[Path] = None
attempted_provider_download = False
provider_sr = None
provider_obj = None
provider_key = self._provider_key_from_item(item)
if provider_key and get_search_plugin and SearchResult:
# Reuse helper to derive the provider key from table/provider/source hints.
provider_obj = get_search_plugin(provider_key, config)
if provider_key and get_provider and SearchResult:
# Reuse helper to derive the plugin key from table/plugin/source hints.
provider_obj = get_provider(provider_key, config)
if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False):
try:
@@ -697,7 +699,7 @@ class Download_File(Cmdlet):
)
continue
# Allow providers to add/enrich tags and metadata during download.
# Allow plugins to add or enrich tags and metadata during download.
if provider_sr is not None:
try:
sr_md = getattr(provider_sr, "full_metadata", None)
@@ -838,9 +840,9 @@ class Download_File(Cmdlet):
notes: Optional[Dict[str, str]] = None
try:
if isinstance(full_metadata, dict):
# Providers attach pre-built notes under the generic "_notes" key
# Plugins attach pre-built notes under the generic "_notes" key
# (e.g. Tidal sets {"lyric": subtitles} during download enrichment).
# This keeps provider-specific metadata handling inside the provider.
# This keeps plugin-specific metadata handling inside the plugin.
_provider_notes = full_metadata.get("_notes")
if isinstance(_provider_notes, dict) and _provider_notes:
notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v}
@@ -949,7 +951,6 @@ class Download_File(Cmdlet):
return {
"get_plugin": getattr(provider_registry, "get_plugin", None),
"get_search_plugin": getattr(provider_registry, "get_search_plugin", None),
"match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
"list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
"SearchResult": SearchResult,
@@ -957,7 +958,6 @@ class Download_File(Cmdlet):
except Exception:
return {
"get_plugin": None,
"get_search_plugin": None,
"match_plugin_name_for_url": None,
"list_selection_url_prefixes": None,
"SearchResult": None,
cmdlet/get_tag.py
+22 -22
View File
@@ -15,10 +15,10 @@ import sys
from SYS.logger import log, debug
from plugins.metadata_provider import (
get_default_subject_scrape_provider,
get_metadata_provider,
get_metadata_provider_for_url,
list_metadata_providers,
get_default_subject_scrape_plugin,
get_metadata_plugin,
get_metadata_plugin_for_url,
list_metadata_plugins,
scrape_isbn_metadata,
scrape_openlibrary_metadata,
)
@@ -393,33 +393,33 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
scrape_url = parsed_args.get("scrape")
scrape_requested = scrape_flag_present or scrape_url is not None
# Handle URL or provider scraping mode.
# Handle URL or metadata-plugin scraping mode.
if scrape_requested:
import json as json_module
scrape_target = str(scrape_url or "").strip() if scrape_url is not None else ""
provider = None
plugin = None
if scrape_target.startswith(("http://", "https://")):
provider = get_metadata_provider_for_url(scrape_target, config)
if provider is None:
log("No metadata provider can scrape this URL", file=sys.stderr)
plugin = get_metadata_plugin_for_url(scrape_target, config)
if plugin is None:
log("No metadata plugin can scrape this URL", file=sys.stderr)
return 1
payload = provider.scrape_url_payload(scrape_target)
payload = plugin.scrape_url_payload(scrape_target)
if not isinstance(payload, dict):
log(f"No metadata extracted from URL via {provider.name}", file=sys.stderr)
log(f"No metadata extracted from URL via {plugin.name}", file=sys.stderr)
return 1
print(json_module.dumps(payload, ensure_ascii=False))
return 0
if scrape_target:
provider = get_metadata_provider(scrape_target, config)
plugin = get_metadata_plugin(scrape_target, config)
else:
provider = get_default_subject_scrape_provider(config)
if provider is None:
plugin = get_default_subject_scrape_plugin(config)
if plugin is None:
if scrape_target:
log(f"Unknown metadata provider: {scrape_target}", file=sys.stderr)
log(f"Unknown metadata plugin: {scrape_target}", file=sys.stderr)
else:
log("No default metadata provider is available for subject scraping", file=sys.stderr)
log("No default metadata plugin is available for subject scraping", file=sys.stderr)
return 1
backend = None
@@ -548,7 +548,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
query_hint = resolved_subject_query or identifier_query or combined_query or title_hint
if not query_hint:
log(
f"No query could be resolved for metadata provider '{provider.name}'",
f"No query could be resolved for metadata plugin '{provider.name}'",
file=sys.stderr
)
return 1
@@ -749,9 +749,9 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
)
return 0
provider_for_apply = get_metadata_provider(str(result_provider), config)
if provider_for_apply is not None:
apply_tags = provider_for_apply.filter_tags_for_store_apply(
plugin_for_apply = get_metadata_plugin(str(result_provider), config)
if plugin_for_apply is not None:
apply_tags = plugin_for_apply.filter_tags_for_store_apply(
[str(t) for t in result_tags if t is not None]
)
else:
@@ -946,7 +946,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
_SCRAPE_CHOICES = []
try:
_SCRAPE_CHOICES = sorted(list_metadata_providers().keys())
_SCRAPE_CHOICES = sorted(list_metadata_plugins().keys())
except Exception:
_SCRAPE_CHOICES = [
"itunes",
@@ -1000,7 +1000,7 @@ class Get_Tag(Cmdlet):
' -query: Override hash to look up in Hydrus (use: -query "hash:<sha256>")',
" -store: Store result to key for downstream pipeline",
" -emit: Quiet mode (no interactive selection)",
" -scrape: Scrape metadata from URL or metadata provider",
" -scrape: Scrape metadata from URL or metadata plugin",
],
exec=self.run,
)
cmdlet/search_file.py
+37 -17
View File
@@ -1,4 +1,4 @@
"""search-file cmdlet: Search for files in storage backends (Hydrus)."""
"""search-file cmdlet: Search store backends and search-capable plugins."""
from __future__ import annotations
@@ -15,11 +15,11 @@ from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import log, debug, debug_panel
from SYS.payload_builders import build_file_result_payload, normalize_file_extension
from ProviderCore.registry import get_search_plugin, list_search_plugins
from ProviderCore.registry import get_plugin_with_capability, list_plugins_with_capability
from SYS.rich_display import (
show_provider_config_panel,
show_plugin_config_panel,
show_store_config_panel,
show_available_providers_panel,
show_available_plugins_panel,
)
from SYS.database import insert_worker, update_worker, append_worker_stdout
from SYS.item_accessors import get_extension_field, get_int_field, get_result_title
@@ -164,13 +164,13 @@ def _summarize_worker_results(results: Sequence[Dict[str, Any]], preview_limit:
class search_file(Cmdlet):
"""Class-based search-file cmdlet for searching storage backends."""
"""Class-based search-file cmdlet for searching backends and providers."""
def __init__(self) -> None:
super().__init__(
name="search-file",
summary="Search storage backends (Hydrus) or external plugins (via -plugin).",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-plugin NAME]",
summary="Search configured store backends or search-capable plugins.",
usage="search-file [-query <query>] [-store BACKEND] [-instance NAME] [-limit N] [-plugin NAME]",
arg=[
CmdletArg(
"limit",
@@ -178,6 +178,7 @@ class search_file(Cmdlet):
description="Limit results (default: 100)"
),
SharedArgs.STORE,
SharedArgs.INSTANCE,
SharedArgs.QUERY,
SharedArgs.PLUGIN,
CmdletArg(
@@ -187,8 +188,10 @@ class search_file(Cmdlet):
),
],
detail=[
"Search across storage backends: Hydrus instances",
"Use -store to search a specific backend by name",
"Search across configured store backends or plugin providers.",
"Use -store to target a specific store backend by name.",
"Use -plugin with -instance to target a named provider config.",
"In plugin mode, -store <name> is kept as a compatibility alias for -instance <name>.",
"URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png",
@@ -207,6 +210,7 @@ class search_file(Cmdlet):
"",
"Plugin search (-plugin):",
"search-file -plugin youtube 'tutorial' # Search YouTube plugin",
"search-file -plugin ftp -instance work '*' # Search a named FTP/SCP plugin instance",
"search-file -plugin alldebrid '*' # List AllDebrid magnets",
"search-file -plugin alldebrid -open 123 '*' # Show files for a magnet",
],
@@ -1451,6 +1455,7 @@ class search_file(Cmdlet):
self,
*,
plugin_name: str,
instance_name: Optional[str],
query: str,
limit: int,
limit_set: bool,
@@ -1475,15 +1480,15 @@ class search_file(Cmdlet):
log("Error: search-file -plugin requires both plugin and query", file=sys.stderr)
log(f"Usage: {self.usage}", file=sys.stderr)
providers_map = list_search_plugins(config)
providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a]
unconfigured = [n for n, a in providers_map.items() if not a]
if unconfigured:
show_provider_config_panel(unconfigured)
show_plugin_config_panel(unconfigured)
if available:
show_available_providers_panel(available)
show_available_plugins_panel(available)
return 1
@@ -1496,7 +1501,7 @@ class search_file(Cmdlet):
if hasattr(ctx_mod, "get_pipeline_state"):
progress = ctx_mod.get_pipeline_state().live_progress
provider = get_search_plugin(plugin_name, config)
provider = get_plugin_with_capability(plugin_name, "search", config)
if not provider:
if progress:
try:
@@ -1504,12 +1509,12 @@ class search_file(Cmdlet):
except Exception:
pass
show_provider_config_panel([plugin_name])
show_plugin_config_panel([plugin_name])
providers_map = list_search_plugins(config)
providers_map = list_plugins_with_capability("search", config)
available = [n for n, a in providers_map.items() if a]
if available:
show_available_providers_panel(available)
show_available_plugins_panel(available)
return 1
worker_id = str(uuid.uuid4())
@@ -1542,6 +1547,8 @@ class search_file(Cmdlet):
normalized_query = (normalized_query or "").strip()
query = normalized_query or "*"
search_filters = dict(provider_filters or {})
if instance_name and not search_filters.get("instance"):
search_filters["instance"] = str(instance_name).strip()
# Dynamic table generation via provider
table_title = provider.get_table_title(query, search_filters).strip().rstrip(":")
@@ -1564,6 +1571,7 @@ class search_file(Cmdlet):
"search-file provider request",
[
("provider", plugin_name),
("instance", search_filters.get("instance") or "<default>"),
("query", query),
("limit", limit),
("filters", search_filters or "<none>"),
@@ -1581,7 +1589,7 @@ class search_file(Cmdlet):
border_style="cyan",
)
# Allow providers to apply provider-specific UX transforms (e.g. auto-expansion)
# Allow plugins to apply plugin-specific UX transforms (e.g. auto-expansion)
try:
post = getattr(provider, "postprocess_search_results", None)
if callable(post) and isinstance(results, list):
@@ -1737,6 +1745,10 @@ class search_file(Cmdlet):
f.lower()
for f in (flag_registry.get("store") or {"-store", "--store"})
}
instance_flags = {
f.lower()
for f in (flag_registry.get("instance") or {"-instance", "--instance"})
}
limit_flags = {
f.lower()
for f in (flag_registry.get("limit") or {"-limit", "--limit"})
@@ -1753,6 +1765,7 @@ class search_file(Cmdlet):
# Parse arguments
query = ""
storage_backend: Optional[str] = None
instance_name: Optional[str] = None
plugin_name: Optional[str] = None
open_id: Optional[int] = None
limit = 100
@@ -1773,6 +1786,10 @@ class search_file(Cmdlet):
plugin_name = args_list[i + 1]
i += 2
continue
if low in instance_flags and i + 1 < len(args_list):
instance_name = args_list[i + 1]
i += 2
continue
if low in open_flags and i + 1 < len(args_list):
try:
open_id = int(args_list[i + 1])
@@ -1804,8 +1821,11 @@ class search_file(Cmdlet):
query = query.strip()
if plugin_name:
if storage_backend and not instance_name:
instance_name = storage_backend
return self._run_plugin_search(
plugin_name=plugin_name,
instance_name=instance_name,
query=query,
limit=limit,
limit_set=limit_set,