huge refactor of plugin system

This commit is contained in:
2026-04-30 18:56:22 -07:00
parent ea3ead248b
commit be5a11da97
99 changed files with 7603 additions and 11320 deletions
+12
View File
@@ -103,6 +103,17 @@ def _register_native_commands() -> None:
pass
def _register_plugin_commands() -> None:
try:
from ProviderCore.commands import register_plugin_commands
except Exception:
return
try:
register_plugin_commands(REGISTRY)
except Exception:
pass
def ensure_cmdlet_modules_loaded(force: bool = False) -> None:
global _MODULES_LOADED
@@ -115,4 +126,5 @@ def ensure_cmdlet_modules_loaded(force: bool = False) -> None:
_load_root_modules()
_load_helper_modules()
_register_native_commands()
_register_plugin_commands()
_MODULES_LOADED = True
+421 -5
View File
@@ -105,9 +105,13 @@ class CmdletArg:
storage_flags = SharedArgs.STORAGE.to_flags()
# Returns: ('--storage', '-storage', '-s')
"""
normalized_name = str(self.name or "").lstrip("-")
if not normalized_name:
return tuple()
flags = [
f"--{self.name}",
f"-{self.name}"
f"--{normalized_name}",
f"-{normalized_name}"
] # Both double-dash and single-dash variants
# Add short form if alias exists
@@ -116,8 +120,8 @@ class CmdletArg:
# Add negation forms for flag type
if self.type == "flag":
flags.append(f"--no-{self.name}")
flags.append(f"-no{self.name}") # Single-dash negation variant
flags.append(f"--no-{normalized_name}")
flags.append(f"-no{normalized_name}") # Single-dash negation variant
if self.alias:
flags.append(f"-n{self.alias}")
@@ -1658,6 +1662,59 @@ def parse_tag_arguments(arguments: Sequence[str]) -> List[str]:
List of normalized tag strings (empty strings filtered out)
"""
def _split_top_level_commas(text: str) -> List[str]:
segments: List[str] = []
current: List[str] = []
paren_depth = 0
angle_depth = 0
quote: Optional[str] = None
escape = False
for ch in text:
if escape:
current.append(ch)
escape = False
continue
if ch == "\\":
current.append(ch)
escape = True
continue
if quote:
current.append(ch)
if ch == quote:
quote = None
continue
if ch in {"'", '"'}:
current.append(ch)
quote = ch
continue
if ch == "(":
paren_depth += 1
current.append(ch)
continue
if ch == ")":
paren_depth = max(0, paren_depth - 1)
current.append(ch)
continue
if ch == "<":
angle_depth += 1
current.append(ch)
continue
if ch == ">":
angle_depth = max(0, angle_depth - 1)
current.append(ch)
continue
if ch == "," and paren_depth == 0 and angle_depth == 0:
segments.append("".join(current).strip())
current = []
continue
current.append(ch)
tail = "".join(current).strip()
if tail or segments:
segments.append(tail)
return segments
def _expand_pipe_namespace(text: str) -> List[str]:
parts = text.split("|")
expanded: List[str] = []
@@ -1684,7 +1741,7 @@ def parse_tag_arguments(arguments: Sequence[str]) -> List[str]:
tags: List[str] = []
for argument in arguments:
for token in argument.split(","):
for token in _split_top_level_commas(str(argument)):
text = token.strip()
if not text:
continue
@@ -1704,6 +1761,365 @@ def parse_tag_arguments(arguments: Sequence[str]) -> List[str]:
return tags
_TAG_VALUE_TEMPLATE_RE = re.compile(r"#\(([^)]+)\)")
_TAG_VALUE_FUNCTION_RE = re.compile(r"<([a-zA-Z_][a-zA-Z0-9_-]*)\((.*?)\)>")
def _normalize_tag_value_template_name(value: Any) -> str:
text = str(value or "").strip().lower()
if not text:
return ""
try:
text = re.sub(r"\s+", " ", text).strip()
except Exception:
text = " ".join(text.split())
return text
def _tag_value_template_keys(value: Any) -> list[str]:
normalized = _normalize_tag_value_template_name(value)
if not normalized:
return []
keys = [normalized]
trimmed_hash = re.sub(r"\s*#+\s*$", "", normalized).strip()
if trimmed_hash and trimmed_hash not in keys:
keys.append(trimmed_hash)
return keys
def _add_tag_values_to_lookup(lookup: Dict[str, List[str]], tag_text: Any) -> None:
text = str(tag_text or "").strip()
if not text or ":" not in text:
return
if _TAG_VALUE_TEMPLATE_RE.search(text) or _TAG_VALUE_FUNCTION_RE.search(text):
return
namespace, value = text.split(":", 1)
value_text = str(value or "").strip()
if not value_text:
return
for key in _tag_value_template_keys(namespace):
values = lookup.setdefault(key, [])
if value_text not in values:
values.append(value_text)
def build_tag_value_lookup(
tags: Optional[Iterable[Any]],
*,
result: Any = None,
) -> Dict[str, List[str]]:
"""Build a placeholder lookup from existing tags and lightweight result fields.
Placeholder lookups use ``#(namespace)`` syntax. Namespace matching is
case-insensitive and trims repeated whitespace. A trailing ``#`` in the
placeholder is ignored so inputs like ``#(track #)`` can resolve ``track:9``.
"""
lookup: Dict[str, List[str]] = {}
for tag in tags or []:
_add_tag_values_to_lookup(lookup, tag)
title_text = extract_title_from_result(result)
if title_text:
_add_tag_values_to_lookup(lookup, f"title:{title_text}")
return lookup
def _split_tag_value_function_args(value: Any) -> list[str]:
text = str(value or "")
args: list[str] = []
current: list[str] = []
depth = 0
quote: Optional[str] = None
escape = False
for ch in text:
if escape:
current.append(ch)
escape = False
continue
if ch == "\\":
current.append(ch)
escape = True
continue
if quote:
current.append(ch)
if ch == quote:
quote = None
continue
if ch in {"'", '"'}:
current.append(ch)
quote = ch
continue
if ch in {"(", "[", "{"}:
depth += 1
current.append(ch)
continue
if ch in {")", "]", "}"}:
depth = max(0, depth - 1)
current.append(ch)
continue
if ch == "," and depth == 0:
args.append("".join(current).strip())
current = []
continue
current.append(ch)
tail = "".join(current).strip()
if tail or args:
args.append(tail)
return args
def _strip_tag_value_function_arg(value: Any) -> str:
text = str(value or "").strip()
if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
return text[1:-1]
return text
def _padding_width_from_spec(value: Any) -> Optional[int]:
spec = _strip_tag_value_function_arg(value)
if not spec:
return None
if re.fullmatch(r"0+", spec):
return len(spec)
if spec.isdigit():
try:
width = int(spec)
except Exception:
return None
return width if width > 0 else None
return None
def _replace_tag_value_placeholders(
value: Any,
lookup: Dict[str, List[str]],
*,
preserve_unresolved: bool,
) -> tuple[str, bool]:
text = str(value or "")
unresolved = False
def _replace(match: re.Match[str]) -> str:
nonlocal unresolved
keys = _tag_value_template_keys(match.group(1) or "")
values: List[str] = []
for key in keys:
for candidate in lookup.get(key, []):
if candidate not in values:
values.append(candidate)
if not values:
unresolved = True
return match.group(0) if preserve_unresolved else ""
return ", ".join(values)
return _TAG_VALUE_TEMPLATE_RE.sub(_replace, text), unresolved
def _coerce_tag_value_integer(value: Any) -> Optional[int]:
text = _strip_tag_value_function_arg(value)
if not text:
return None
if not re.fullmatch(r"[+-]?\d+", text):
return None
try:
return int(text)
except Exception:
return None
def _apply_tag_value_function(
name: str,
args: Sequence[str],
*,
lookup: Dict[str, List[str]],
) -> Optional[str]:
func = str(name or "").strip().lower()
resolved_values: list[str] = []
unresolved_flags: list[bool] = []
for arg in args:
rendered, unresolved = _replace_tag_value_placeholders(
arg,
lookup,
preserve_unresolved=True,
)
resolved_values.append(_strip_tag_value_function_arg(rendered))
unresolved_flags.append(unresolved)
if func in {"padding", "pad", "zfill"}:
if len(resolved_values) != 2 or any(unresolved_flags):
return None
width = _padding_width_from_spec(resolved_values[0])
if width is None:
return None
return str(resolved_values[1]).zfill(width)
if func == "default":
if len(resolved_values) != 2:
return None
primary = resolved_values[0]
fallback = resolved_values[1]
if not unresolved_flags[0] and str(primary).strip():
return str(primary)
if unresolved_flags[1]:
return None
return str(fallback)
if func == "replace":
if len(resolved_values) != 3 or any(unresolved_flags):
return None
return str(resolved_values[0]).replace(
str(resolved_values[1]),
str(resolved_values[2]),
)
if func in {"increment", "inc", "add"}:
if len(resolved_values) not in {1, 2}:
return None
if unresolved_flags[0]:
return None
base_value = _coerce_tag_value_integer(resolved_values[0])
if base_value is None:
return None
step_value = 1
if len(resolved_values) == 2:
if unresolved_flags[1]:
return None
parsed_step = _coerce_tag_value_integer(resolved_values[1])
if parsed_step is None:
return None
step_value = parsed_step
return str(base_value + step_value)
return None
def _render_tag_value_function_templates(
value: Any,
*,
lookup: Dict[str, List[str]],
) -> tuple[str, bool]:
text = str(value or "")
unresolved = False
def _replace(match: re.Match[str]) -> str:
nonlocal unresolved
func_name = match.group(1) or ""
func_args = _split_tag_value_function_args(match.group(2) or "")
rendered = _apply_tag_value_function(
func_name,
func_args,
lookup=lookup,
)
if rendered is None:
unresolved = True
return match.group(0)
return rendered
previous = None
rendered = text
while previous != rendered and _TAG_VALUE_FUNCTION_RE.search(rendered):
previous = rendered
rendered = _TAG_VALUE_FUNCTION_RE.sub(_replace, rendered)
if unresolved:
break
return rendered, unresolved
def render_tag_value_templates(
tags: Sequence[Any],
*,
existing_tags: Optional[Iterable[Any]] = None,
result: Any = None,
) -> tuple[list[str], list[str]]:
"""Resolve ``#(namespace)`` placeholders and ``<transform(...)>`` functions.
Returns ``(resolved_tags, unresolved_templates)``. Tags whose placeholders
cannot be fully resolved are omitted from ``resolved_tags`` and returned in
``unresolved_templates`` so callers can warn or summarize skipped items.
Currently supported transforms:
- ``<padding(00,#(episode))>`` or ``<pad(2,#(episode))>`` for zero-padding
- ``<default(#(season),0)>`` to fall back when a placeholder is missing
- ``<replace(#(title),old,new)>`` for simple substring replacement
- ``<increment(#(episode),1)>`` for integer arithmetic
"""
entries: list[dict[str, Any]] = []
lookup = build_tag_value_lookup(existing_tags, result=result)
for raw_tag in tags or []:
text = str(raw_tag or "").strip()
if not text:
continue
has_template = bool(
_TAG_VALUE_TEMPLATE_RE.search(text)
or _TAG_VALUE_FUNCTION_RE.search(text)
)
entry = {
"raw": text,
"resolved": None,
"has_template": has_template,
}
if not has_template:
entry["resolved"] = text
_add_tag_values_to_lookup(lookup, text)
entries.append(entry)
progress = True
while progress:
progress = False
for entry in entries:
if entry["resolved"] is not None or not entry["has_template"]:
continue
rendered, unresolved = _replace_tag_value_placeholders(
entry["raw"],
lookup,
preserve_unresolved=bool(_TAG_VALUE_FUNCTION_RE.search(str(entry["raw"]))),
)
rendered, function_unresolved = _render_tag_value_function_templates(
rendered,
lookup=lookup,
)
if function_unresolved:
continue
if unresolved and _TAG_VALUE_TEMPLATE_RE.search(rendered):
continue
rendered = rendered.strip()
if not rendered:
entry["resolved"] = ""
progress = True
continue
entry["resolved"] = rendered
_add_tag_values_to_lookup(lookup, rendered)
progress = True
resolved_tags = merge_sequences(
[entry["resolved"] for entry in entries if isinstance(entry.get("resolved"), str) and entry.get("resolved")],
case_sensitive=True,
)
unresolved_templates = [
str(entry["raw"])
for entry in entries
if entry["has_template"] and not entry.get("resolved")
]
return resolved_tags, unresolved_templates
def fmt_bytes(n: Optional[int]) -> str:
"""Format bytes as human-readable with 1 decimal place (MB/GB).
+111 -19
View File
@@ -337,6 +337,13 @@ class Add_File(Cmdlet):
except Exception as exc:
debug(f"[add-file] Directory scan failed: {exc}")
if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results:
try:
if ctx.get_stage_context() is not None:
return 0
except Exception:
pass
# Determine if -store targets a registered backend (vs a filesystem export path).
is_storage_backend_location = False
if location:
@@ -354,6 +361,19 @@ class Add_File(Cmdlet):
)
return 1
plugin_storage_backend = None
if plugin_name:
plugin_storage_backend = Add_File._resolve_plugin_storage_backend(
plugin_name,
plugin_instance,
config,
store_instance=storage_registry,
)
effective_storage_backend_name = plugin_storage_backend or (
str(location) if location and is_storage_backend_location else None
)
# Decide which items to process.
# - If directory scan was performed, use those results
# - If user provided -path (and it was not reinterpreted as destination), treat this invocation as single-item.
@@ -371,13 +391,6 @@ class Add_File(Cmdlet):
else:
items_to_process = [result]
if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results:
try:
if ctx.get_stage_context() is not None:
return 0
except Exception:
pass
total_items = len(items_to_process) if isinstance(items_to_process, list) else 0
processed_items = 0
try:
@@ -549,15 +562,16 @@ class Add_File(Cmdlet):
live_progress = None
want_final_search_file = (
bool(is_last_stage) and bool(is_storage_backend_location)
and bool(location) and bool(live_progress)
bool(is_last_stage)
and bool(effective_storage_backend_name)
and bool(live_progress)
)
auto_search_file_after_add = False
# When ingesting multiple items into a backend store, defer URL association and
# apply it once at the end (bulk) to avoid per-item URL API calls.
defer_url_association = (
bool(is_storage_backend_location) and bool(location)
bool(effective_storage_backend_name)
and len(items_to_process) > 1
)
@@ -642,7 +656,7 @@ class Add_File(Cmdlet):
# When using -path (filesystem export), allow all file types.
# When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
allow_all_files = not (location and is_storage_backend_location)
allow_all_files = not bool(effective_storage_backend_name)
if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
failures += 1
continue
@@ -653,14 +667,33 @@ class Add_File(Cmdlet):
progress.step("ingesting file")
if plugin_name:
code = self._handle_plugin_upload(
media_path,
plugin_name,
plugin_instance,
pipe_obj,
config,
delete_after_item
)
if effective_storage_backend_name:
code = self._handle_storage_backend(
item,
media_path,
effective_storage_backend_name,
pipe_obj,
config,
delete_after_item,
collect_payloads=collected_payloads,
collect_relationship_pairs=pending_relationship_pairs,
defer_url_association=defer_url_association,
pending_url_associations=pending_url_associations,
defer_tag_association=defer_url_association,
pending_tag_associations=pending_tag_associations,
suppress_last_stage_overlay=want_final_search_file,
auto_search_file=auto_search_file_after_add,
store_instance=storage_registry,
)
else:
code = self._handle_plugin_upload(
media_path,
plugin_name,
plugin_instance,
pipe_obj,
config,
delete_after_item
)
if code == 0:
successes += 1
else:
@@ -1431,6 +1464,65 @@ class Add_File(Cmdlet):
normalized = normalized.split(".", 1)[0]
return normalized
@staticmethod
def _resolve_plugin_storage_backend(
plugin_name: Optional[Any],
instance_name: Optional[Any],
config: Dict[str, Any],
*,
store_instance: Optional[Any] = None,
) -> Optional[str]:
plugin_key = Add_File._normalize_provider_key(plugin_name)
if not plugin_key:
return None
from ProviderCore.registry import get_plugin_with_capability
file_provider = get_plugin_with_capability(plugin_key, "upload", config)
if file_provider is None:
return None
resolver = getattr(file_provider, "resolve_backend", None)
if not callable(resolver):
return None
explicit_instance = str(instance_name or "").strip() or None
try:
storage = store_instance if store_instance is not None else Store(config)
except Exception:
storage = None
try:
resolved_name, backend = resolver(
explicit_instance,
storage=storage,
require_explicit=bool(explicit_instance),
)
except TypeError:
try:
resolved_name, backend = resolver(explicit_instance)
except Exception:
return None
except Exception:
return None
if backend is None:
return None
resolved_text = str(resolved_name or explicit_instance or "").strip()
if not resolved_text:
return None
checker = getattr(file_provider, "is_backend", None)
if callable(checker):
try:
if not checker(backend, resolved_text):
return None
except Exception:
return None
return resolved_text
@staticmethod
def _maybe_download_plugin_result(
result: Any,
+28
View File
@@ -22,6 +22,8 @@ SharedArgs = sh.SharedArgs
normalize_hash = sh.normalize_hash
parse_tag_arguments = sh.parse_tag_arguments
expand_tag_groups = sh.expand_tag_groups
merge_sequences = sh.merge_sequences
render_tag_value_templates = sh.render_tag_value_templates
parse_cmdlet_args = sh.parse_cmdlet_args
collapse_namespace_tag = sh.collapse_namespace_tag
should_show_help = sh.should_show_help
@@ -524,6 +526,11 @@ class Add_Tag(Cmdlet):
"- The source namespace must already exist in the file being tagged.",
"- Target namespaces that already have a value are skipped (not overwritten).",
"- Use -extract to derive namespaced tags from the current title (title field or title: tag) using a simple template.",
"- Use #(namespace) inside a tag value to insert existing values, e.g. add-tag \"title:#(track) - #(series)\".",
"- Use angle-bracket transforms for advanced formatting, e.g. add-tag \"code:e<padding(00,#(episode))>\".",
"- Current documented transforms include padding, default, replace, and increment.",
"- Template examples assume lowercase tag text; case transforms are intentionally not part of the documented syntax.",
"- See docs/tag_template_syntax.md for recipe-style examples and the current shared template syntax.",
],
exec=self.run,
)
@@ -655,6 +662,7 @@ class Add_Tag(Cmdlet):
# tag ARE provided - apply them to each store-backed result
total_added = 0
total_modified = 0
unresolved_template_count = 0
store_registry = Store(config, suppress_debug=True)
@@ -791,6 +799,13 @@ class Add_Tag(Cmdlet):
if new_tag.lower() not in existing_lower:
item_tag_to_add.append(new_tag)
item_tag_to_add, unresolved_templates = render_tag_value_templates(
item_tag_to_add,
existing_tags=merge_sequences(existing_tag_list, item_tag_to_add, case_sensitive=True),
result=res,
)
unresolved_template_count += len(unresolved_templates)
item_tag_to_add = collapse_namespace_tag(
item_tag_to_add,
"title",
@@ -962,6 +977,13 @@ class Add_Tag(Cmdlet):
if new_tag.lower() not in existing_lower:
item_tag_to_add.append(new_tag)
item_tag_to_add, unresolved_templates = render_tag_value_templates(
item_tag_to_add,
existing_tags=merge_sequences(existing_tag_list, item_tag_to_add, case_sensitive=True),
result=res,
)
unresolved_template_count += len(unresolved_templates)
item_tag_to_add = collapse_namespace_tag(
item_tag_to_add,
"title",
@@ -1109,6 +1131,12 @@ class Add_Tag(Cmdlet):
file=sys.stderr,
)
if unresolved_template_count > 0:
log(
f"[add_tag] skipped {unresolved_template_count} tag template(s) with unresolved #(namespace) placeholders",
file=sys.stderr,
)
return 0
+34 -3
View File
@@ -11,6 +11,9 @@ CmdletArg = sh.CmdletArg
SharedArgs = sh.SharedArgs
normalize_hash = sh.normalize_hash
parse_tag_arguments = sh.parse_tag_arguments
render_tag_value_templates = sh.render_tag_value_templates
merge_sequences = sh.merge_sequences
extract_tag_from_result = sh.extract_tag_from_result
should_show_help = sh.should_show_help
get_field = sh.get_field
from SYS.logger import debug, log
@@ -133,6 +136,11 @@ CMDLET = Cmdlet(
detail=[
"- Requires a Hydrus file (hash present) or explicit -query override.",
"- Multiple tags can be comma-separated or space-separated.",
"- Use #(namespace) inside a tag value to remove a derived tag, e.g. delete-tag \"title:#(track) - #(series)\".",
"- Angle-bracket transforms match add-tag syntax, e.g. delete-tag \"code:e<padding(00,#(episode))>\".",
"- Current documented transforms include padding, default, replace, and increment.",
"- Template examples assume lowercase tag text; case transforms are intentionally not part of the documented syntax.",
"- See docs/tag_template_syntax.md for recipe-style examples and the current shared template syntax.",
],
)
@@ -225,7 +233,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
store_name = override_store or get_field(result, "store")
path = get_field(result, "path") or get_field(result, "target")
tags = [str(t) for t in grouped_tags if t]
return 0 if _process_deletion(tags, file_hash, path, store_name, config) else 1
return 0 if _process_deletion(tags, file_hash, path, store_name, config, result=result) else 1
if not tags_arg and not has_piped_tag and not has_piped_tag_list:
log("Requires at least one tag argument")
@@ -316,7 +324,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
item_hash,
item_path,
item_store,
config):
config,
result=item):
success_count += 1
if success_count > 0:
@@ -331,6 +340,7 @@ def _process_deletion(
store_name: str | None,
config: Dict[str,
Any],
result: Any = None,
) -> bool:
"""Helper to execute the deletion logic for a single target."""
@@ -367,12 +377,33 @@ def _process_deletion(
except Exception:
return []
existing_tag_list = merge_sequences(
extract_tag_from_result(result),
_fetch_existing_tags(),
case_sensitive=True,
)
resolved_tags, unresolved_templates = render_tag_value_templates(
tags,
existing_tags=existing_tag_list,
result=result,
)
if unresolved_templates:
log(
f"[delete_tag] skipped {len(unresolved_templates)} tag template(s) with unresolved #(namespace) placeholders",
file=sys.stderr,
)
tags = list(resolved_tags)
if not tags:
return False
# Safety: only block if this deletion would remove the final title tag
title_tags = [
t for t in tags if isinstance(t, str) and t.lower().startswith("title:")
]
if title_tags:
existing_tags = _fetch_existing_tags()
existing_tags = existing_tag_list
current_titles = [
t for t in existing_tags
if isinstance(t, str) and t.lower().startswith("title:")
+4 -4
View File
@@ -1231,7 +1231,7 @@ class search_file(Cmdlet):
log(f"No web results found for query: {search_query}", file=sys.stderr)
if refresh_mode:
try:
ctx.set_last_result_table_preserve_history(table, [])
ctx.set_last_result_table_overlay(table, [])
except Exception:
pass
try:
@@ -2089,7 +2089,7 @@ class search_file(Cmdlet):
pass
if refresh_mode:
ctx.set_last_result_table_preserve_history(
ctx.set_last_result_table_overlay(
table,
results_list
)
@@ -2106,7 +2106,7 @@ class search_file(Cmdlet):
if refresh_mode:
try:
table.title = command_title
ctx.set_last_result_table_preserve_history(table, [])
ctx.set_last_result_table_overlay(table, [])
except Exception:
pass
db.append_worker_stdout(worker_id, _summarize_worker_results([]))
@@ -2279,7 +2279,7 @@ class search_file(Cmdlet):
if refresh_mode:
try:
table.title = command_title
ctx.set_last_result_table_preserve_history(table, [])
ctx.set_last_result_table_overlay(table, [])
except Exception:
pass
db.append_worker_stdout(worker_id, _summarize_worker_results([]))